drivers/infiniband/sw/siw/siw_qp.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2
   3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4 /* Copyright (c) 2008-2019, IBM Corporation */
   5
   6 #include <linux/errno.h>
   7 #include <linux/types.h>
   8 #include <linux/net.h>
   9 #include <linux/scatterlist.h>
  10 #include <linux/llist.h>
  11 #include <asm/barrier.h>
  12 #include <net/tcp.h>
  13
  14 #include "siw.h"
  15 #include "siw_verbs.h"
  16 #include "siw_mem.h"
  17
  18 static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = {
  19         [SIW_QP_STATE_IDLE] = "IDLE",
  20         [SIW_QP_STATE_RTR] = "RTR",
  21         [SIW_QP_STATE_RTS] = "RTS",
  22         [SIW_QP_STATE_CLOSING] = "CLOSING",
  23         [SIW_QP_STATE_TERMINATE] = "TERMINATE",
  24         [SIW_QP_STATE_ERROR] = "ERROR"
  25 };
  26
  27 /*
  28  * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a
  29  * per-RDMAP message basis. Please keep order of initializer. All MPA len
  30  * is initialized to minimum packet size.
  31  */
  32 struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = {
  33         { /* RDMAP_RDMA_WRITE */
  34           .hdr_len = sizeof(struct iwarp_rdma_write),
  35           .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
  36           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
  37                                  cpu_to_be16(DDP_VERSION << 8) |
  38                                  cpu_to_be16(RDMAP_VERSION << 6) |
  39                                  cpu_to_be16(RDMAP_RDMA_WRITE),
  40           .rx_data = siw_proc_write },
  41         { /* RDMAP_RDMA_READ_REQ */
  42           .hdr_len = sizeof(struct iwarp_rdma_rreq),
  43           .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
  44           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  45                                  cpu_to_be16(RDMAP_VERSION << 6) |
  46                                  cpu_to_be16(RDMAP_RDMA_READ_REQ),
  47           .rx_data = siw_proc_rreq },
  48         { /* RDMAP_RDMA_READ_RESP */
  49           .hdr_len = sizeof(struct iwarp_rdma_rresp),
  50           .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
  51           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
  52                                  cpu_to_be16(DDP_VERSION << 8) |
  53                                  cpu_to_be16(RDMAP_VERSION << 6) |
  54                                  cpu_to_be16(RDMAP_RDMA_READ_RESP),
  55           .rx_data = siw_proc_rresp },
  56         { /* RDMAP_SEND */
  57           .hdr_len = sizeof(struct iwarp_send),
  58           .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
  59           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  60                                  cpu_to_be16(RDMAP_VERSION << 6) |
  61                                  cpu_to_be16(RDMAP_SEND),
  62           .rx_data = siw_proc_send },
  63         { /* RDMAP_SEND_INVAL */
  64           .hdr_len = sizeof(struct iwarp_send_inv),
  65           .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
  66           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  67                                  cpu_to_be16(RDMAP_VERSION << 6) |
  68                                  cpu_to_be16(RDMAP_SEND_INVAL),
  69           .rx_data = siw_proc_send },
  70         { /* RDMAP_SEND_SE */
  71           .hdr_len = sizeof(struct iwarp_send),
  72           .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
  73           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  74                                  cpu_to_be16(RDMAP_VERSION << 6) |
  75                                  cpu_to_be16(RDMAP_SEND_SE),
  76           .rx_data = siw_proc_send },
  77         { /* RDMAP_SEND_SE_INVAL */
  78           .hdr_len = sizeof(struct iwarp_send_inv),
  79           .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
  80           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  81                                  cpu_to_be16(RDMAP_VERSION << 6) |
  82                                  cpu_to_be16(RDMAP_SEND_SE_INVAL),
  83           .rx_data = siw_proc_send },
  84         { /* RDMAP_TERMINATE */
  85           .hdr_len = sizeof(struct iwarp_terminate),
  86           .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
  87           .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
  88                                  cpu_to_be16(RDMAP_VERSION << 6) |
  89                                  cpu_to_be16(RDMAP_TERMINATE),
  90           .rx_data = siw_proc_terminate }
  91 };
  92
  93 void siw_qp_llp_data_ready(struct sock *sk)
  94 {
  95         struct siw_qp *qp;
  96
  97         read_lock(&sk->sk_callback_lock);
  98
  99         if (unlikely(!sk->sk_user_data || !sk_to_qp(sk)))
 100                 goto done;
 101
 102         qp = sk_to_qp(sk);
 103
 104         if (likely(!qp->rx_stream.rx_suspend &&
 105                    down_read_trylock(&qp->state_lock))) {
 106                 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };
 107
 108                 if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
 109                         /*
 110                          * Implements data receive operation during
 111                          * socket callback. TCP gracefully catches
 112                          * the case where there is nothing to receive
 113                          * (not calling siw_tcp_rx_data() then).
 114                          */
 115                         tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
 116
 117                 up_read(&qp->state_lock);
 118         } else {
 119                 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",
 120                            qp->rx_stream.rx_suspend);
 121         }
 122 done:
 123         read_unlock(&sk->sk_callback_lock);
 124 }
 125
 126 void siw_qp_llp_close(struct siw_qp *qp)
 127 {
 128         siw_dbg_qp(qp, "enter llp close, state = %s\n",
 129                    siw_qp_state_to_string[qp->attrs.state]);
 130
 131         down_write(&qp->state_lock);
 132
 133         qp->rx_stream.rx_suspend = 1;
 134         qp->tx_ctx.tx_suspend = 1;
 135         qp->attrs.sk = NULL;
 136
 137         switch (qp->attrs.state) {
 138         case SIW_QP_STATE_RTS:
 139         case SIW_QP_STATE_RTR:
 140         case SIW_QP_STATE_IDLE:
 141         case SIW_QP_STATE_TERMINATE:
 142                 qp->attrs.state = SIW_QP_STATE_ERROR;
 143                 break;
 144         /*
 145          * SIW_QP_STATE_CLOSING:
 146          *
 147          * This is a forced close. shall the QP be moved to
 148          * ERROR or IDLE ?
 149          */
 150         case SIW_QP_STATE_CLOSING:
 151                 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
 152                         qp->attrs.state = SIW_QP_STATE_ERROR;
 153                 else
 154                         qp->attrs.state = SIW_QP_STATE_IDLE;
 155                 break;
 156
 157         default:
 158                 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",
 159                            siw_qp_state_to_string[qp->attrs.state]);
 160                 break;
 161         }
 162         siw_sq_flush(qp);
 163         siw_rq_flush(qp);
 164
 165         /*
 166          * Dereference closing CEP
 167          */
 168         if (qp->cep) {
 169                 siw_cep_put(qp->cep);
 170                 qp->cep = NULL;
 171         }
 172
 173         up_write(&qp->state_lock);
 174
 175         siw_dbg_qp(qp, "llp close exit: state %s\n",
 176                    siw_qp_state_to_string[qp->attrs.state]);
 177 }
 178
 179 /*
 180  * socket callback routine informing about newly available send space.
 181  * Function schedules SQ work for processing SQ items.
 182  */
 183 void siw_qp_llp_write_space(struct sock *sk)
 184 {
 185         struct siw_cep *cep;
 186
 187         read_lock(&sk->sk_callback_lock);
 188
 189         cep  = sk_to_cep(sk);
 190         if (cep) {
 191                 cep->sk_write_space(sk);
 192
 193                 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
 194                         (void)siw_sq_start(cep->qp);
 195         }
 196
 197         read_unlock(&sk->sk_callback_lock);
 198 }
 199
 200 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)
 201 {
 202         irq_size = roundup_pow_of_two(irq_size);
 203         orq_size = roundup_pow_of_two(orq_size);
 204
 205         qp->attrs.irq_size = irq_size;
 206         qp->attrs.orq_size = orq_size;
 207
 208         qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe));
 209         if (!qp->irq) {
 210                 siw_dbg_qp(qp, "irq malloc for %d failed\n", irq_size);
 211                 qp->attrs.irq_size = 0;
 212                 return -ENOMEM;
 213         }
 214         qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe));
 215         if (!qp->orq) {
 216                 siw_dbg_qp(qp, "orq malloc for %d failed\n", orq_size);
 217                 qp->attrs.orq_size = 0;
 218                 qp->attrs.irq_size = 0;
 219                 vfree(qp->irq);
 220                 return -ENOMEM;
 221         }
 222         siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);
 223         return 0;
 224 }
 225
 226 static int siw_qp_enable_crc(struct siw_qp *qp)
 227 {
 228         struct siw_rx_stream *c_rx = &qp->rx_stream;
 229         struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
 230         int size;
 231
 232         if (siw_crypto_shash == NULL)
 233                 return -ENOENT;
 234
 235         size = crypto_shash_descsize(siw_crypto_shash) +
 236                 sizeof(struct shash_desc);
 237
 238         c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL);
 239         c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL);
 240         if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) {
 241                 kfree(c_tx->mpa_crc_hd);
 242                 kfree(c_rx->mpa_crc_hd);
 243                 c_tx->mpa_crc_hd = NULL;
 244                 c_rx->mpa_crc_hd = NULL;
 245                 return -ENOMEM;
 246         }
 247         c_tx->mpa_crc_hd->tfm = siw_crypto_shash;
 248         c_rx->mpa_crc_hd->tfm = siw_crypto_shash;
 249
 250         return 0;
 251 }
 252
 253 /*
 254  * Send a non signalled READ or WRITE to peer side as negotiated
 255  * with MPAv2 P2P setup protocol. The work request is only created
 256  * as a current active WR and does not consume Send Queue space.
 257  *
 258  * Caller must hold QP state lock.
 259  */
 260 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)
 261 {
 262         struct siw_wqe *wqe = tx_wqe(qp);
 263         unsigned long flags;
 264         int rv = 0;
 265
 266         spin_lock_irqsave(&qp->sq_lock, flags);
 267
 268         if (unlikely(wqe->wr_status != SIW_WR_IDLE)) {
 269                 spin_unlock_irqrestore(&qp->sq_lock, flags);
 270                 return -EIO;
 271         }
 272         memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
 273
 274         wqe->wr_status = SIW_WR_QUEUED;
 275         wqe->sqe.flags = 0;
 276         wqe->sqe.num_sge = 1;
 277         wqe->sqe.sge[0].length = 0;
 278         wqe->sqe.sge[0].laddr = 0;
 279         wqe->sqe.sge[0].lkey = 0;
 280         /*
 281          * While it must not be checked for inbound zero length
 282          * READ/WRITE, some HW may treat STag 0 special.
 283          */
 284         wqe->sqe.rkey = 1;
 285         wqe->sqe.raddr = 0;
 286         wqe->processed = 0;
 287
 288         if (ctrl & MPA_V2_RDMA_WRITE_RTR)
 289                 wqe->sqe.opcode = SIW_OP_WRITE;
 290         else if (ctrl & MPA_V2_RDMA_READ_RTR) {
 291                 struct siw_sqe *rreq;
 292
 293                 wqe->sqe.opcode = SIW_OP_READ;
 294
 295                 spin_lock(&qp->orq_lock);
 296
 297                 rreq = orq_get_free(qp);
 298                 if (rreq) {
 299                         siw_read_to_orq(rreq, &wqe->sqe);
 300                         qp->orq_put++;
 301                 } else
 302                         rv = -EIO;
 303
 304                 spin_unlock(&qp->orq_lock);
 305         } else
 306                 rv = -EINVAL;
 307
 308         if (rv)
 309                 wqe->wr_status = SIW_WR_IDLE;
 310
 311         spin_unlock_irqrestore(&qp->sq_lock, flags);
 312
 313         if (!rv)
 314                 rv = siw_sq_start(qp);
 315
 316         return rv;
 317 }
 318
 319 /*
 320  * Map memory access error to DDP tagged error
 321  */
 322 enum ddp_ecode siw_tagged_error(enum siw_access_state state)
 323 {
 324         switch (state) {
 325         case E_STAG_INVALID:
 326                 return DDP_ECODE_T_INVALID_STAG;
 327         case E_BASE_BOUNDS:
 328                 return DDP_ECODE_T_BASE_BOUNDS;
 329         case E_PD_MISMATCH:
 330                 return DDP_ECODE_T_STAG_NOT_ASSOC;
 331         case E_ACCESS_PERM:
 332                 /*
 333                  * RFC 5041 (DDP) lacks an ecode for insufficient access
 334                  * permissions. 'Invalid STag' seem to be the closest
 335                  * match though.
 336                  */
 337                 return DDP_ECODE_T_INVALID_STAG;
 338         default:
 339                 WARN_ON(1);
 340                 return DDP_ECODE_T_INVALID_STAG;
 341         }
 342 }
 343
 344 /*
 345  * Map memory access error to RDMAP protection error
 346  */
 347 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state)
 348 {
 349         switch (state) {
 350         case E_STAG_INVALID:
 351                 return RDMAP_ECODE_INVALID_STAG;
 352         case E_BASE_BOUNDS:
 353                 return RDMAP_ECODE_BASE_BOUNDS;
 354         case E_PD_MISMATCH:
 355                 return RDMAP_ECODE_STAG_NOT_ASSOC;
 356         case E_ACCESS_PERM:
 357                 return RDMAP_ECODE_ACCESS_RIGHTS;
 358         default:
 359                 return RDMAP_ECODE_UNSPECIFIED;
 360         }
 361 }
 362
 363 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,
 364                         u8 ecode, int in_tx)
 365 {
 366         if (!qp->term_info.valid) {
 367                 memset(&qp->term_info, 0, sizeof(qp->term_info));
 368                 qp->term_info.layer = layer;
 369                 qp->term_info.etype = etype;
 370                 qp->term_info.ecode = ecode;
 371                 qp->term_info.in_tx = in_tx;
 372                 qp->term_info.valid = 1;
 373         }
 374         siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
 375                    layer, etype, ecode, in_tx ? "yes" : "no");
 376 }
 377
 378 /*
 379  * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581.
 380  * Sending TERMINATE messages is best effort - such messages
 381  * can only be send if the QP is still connected and it does
 382  * not have another outbound message in-progress, i.e. the
 383  * TERMINATE message must not interfer with an incomplete current
 384  * transmit operation.
 385  */
 386 void siw_send_terminate(struct siw_qp *qp)
 387 {
 388         struct kvec iov[3];
 389         struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
 390         struct iwarp_terminate *term = NULL;
 391         union iwarp_hdr *err_hdr = NULL;
 392         struct socket *s = qp->attrs.sk;
 393         struct siw_rx_stream *srx = &qp->rx_stream;
 394         union iwarp_hdr *rx_hdr = &srx->hdr;
 395         u32 crc = 0;
 396         int num_frags, len_terminate, rv;
 397
 398         if (!qp->term_info.valid)
 399                 return;
 400
 401         qp->term_info.valid = 0;
 402
 403         if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
 404                 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",
 405                            tx_type(tx_wqe(qp)));
 406                 return;
 407         }
 408         if (!s && qp->cep)
 409                 /* QP not yet in RTS. Take socket from connection end point */
 410                 s = qp->cep->sock;
 411
 412         if (!s) {
 413                 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");
 414                 return;
 415         }
 416
 417         term = kzalloc(sizeof(*term), GFP_KERNEL);
 418         if (!term)
 419                 return;
 420
 421         term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE);
 422         term->ddp_mo = 0;
 423         term->ddp_msn = cpu_to_be32(1);
 424
 425         iov[0].iov_base = term;
 426         iov[0].iov_len = sizeof(*term);
 427
 428         if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
 429             ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
 430              (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
 431                 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL);
 432                 if (!err_hdr) {
 433                         kfree(term);
 434                         return;
 435                 }
 436         }
 437         memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl,
 438                sizeof(struct iwarp_ctrl));
 439
 440         __rdmap_term_set_layer(term, qp->term_info.layer);
 441         __rdmap_term_set_etype(term, qp->term_info.etype);
 442         __rdmap_term_set_ecode(term, qp->term_info.ecode);
 443
 444         switch (qp->term_info.layer) {
 445         case TERM_ERROR_LAYER_RDMAP:
 446                 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
 447                         /* No additional DDP/RDMAP header to be included */
 448                         break;
 449
 450                 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
 451                         /*
 452                          * Complete RDMAP frame will get attached, and
 453                          * DDP segment length is valid
 454                          */
 455                         term->flag_m = 1;
 456                         term->flag_d = 1;
 457                         term->flag_r = 1;
 458
 459                         if (qp->term_info.in_tx) {
 460                                 struct iwarp_rdma_rreq *rreq;
 461                                 struct siw_wqe *wqe = tx_wqe(qp);
 462
 463                                 /* Inbound RREQ error, detected during
 464                                  * RRESP creation. Take state from
 465                                  * current TX work queue element to
 466                                  * reconstruct peers RREQ.
 467                                  */
 468                                 rreq = (struct iwarp_rdma_rreq *)err_hdr;
 469
 470                                 memcpy(&rreq->ctrl,
 471                                        &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
 472                                        sizeof(struct iwarp_ctrl));
 473
 474                                 rreq->rsvd = 0;
 475                                 rreq->ddp_qn =
 476                                         htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
 477
 478                                 /* Provide RREQ's MSN as kept aside */
 479                                 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);
 480
 481                                 rreq->ddp_mo = htonl(wqe->processed);
 482                                 rreq->sink_stag = htonl(wqe->sqe.rkey);
 483                                 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);
 484                                 rreq->read_size = htonl(wqe->sqe.sge[0].length);
 485                                 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);
 486                                 rreq->source_to =
 487                                         cpu_to_be64(wqe->sqe.sge[0].laddr);
 488
 489                                 iov[1].iov_base = rreq;
 490                                 iov[1].iov_len = sizeof(*rreq);
 491
 492                                 rx_hdr = (union iwarp_hdr *)rreq;
 493                         } else {
 494                                 /* Take RDMAP/DDP information from
 495                                  * current (failed) inbound frame.
 496                                  */
 497                                 iov[1].iov_base = rx_hdr;
 498
 499                                 if (__rdmap_get_opcode(&rx_hdr->ctrl) ==
 500                                     RDMAP_RDMA_READ_REQ)
 501                                         iov[1].iov_len =
 502                                                 sizeof(struct iwarp_rdma_rreq);
 503                                 else /* SEND type */
 504                                         iov[1].iov_len =
 505                                                 sizeof(struct iwarp_send);
 506                         }
 507                 } else {
 508                         /* Do not report DDP hdr information if packet
 509                          * layout is unknown
 510                          */
 511                         if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
 512                             (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
 513                                 break;
 514
 515                         iov[1].iov_base = rx_hdr;
 516
 517                         /* Only DDP frame will get attached */
 518                         if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
 519                                 iov[1].iov_len =
 520                                         sizeof(struct iwarp_rdma_write);
 521                         else
 522                                 iov[1].iov_len = sizeof(struct iwarp_send);
 523
 524                         term->flag_m = 1;
 525                         term->flag_d = 1;
 526                 }
 527                 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len);
 528                 break;
 529
 530         case TERM_ERROR_LAYER_DDP:
 531                 /* Report error encountered while DDP processing.
 532                  * This can only happen as a result of inbound
 533                  * DDP processing
 534                  */
 535
 536                 /* Do not report DDP hdr information if packet
 537                  * layout is unknown
 538                  */
 539                 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
 540                      (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
 541                     ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
 542                      (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
 543                         break;
 544
 545                 iov[1].iov_base = rx_hdr;
 546
 547                 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
 548                         iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged);
 549                 else
 550                         iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged);
 551
 552                 term->flag_m = 1;
 553                 term->flag_d = 1;
 554                 break;
 555
 556         default:
 557                 break;
 558         }
 559         if (term->flag_m || term->flag_d || term->flag_r) {
 560                 iov[2].iov_base = &crc;
 561                 iov[2].iov_len = sizeof(crc);
 562                 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE;
 563                 num_frags = 3;
 564         } else {
 565                 iov[1].iov_base = &crc;
 566                 iov[1].iov_len = sizeof(crc);
 567                 len_terminate = sizeof(*term) + MPA_CRC_SIZE;
 568                 num_frags = 2;
 569         }
 570
 571         /* Adjust DDP Segment Length parameter, if valid */
 572         if (term->flag_m) {
 573                 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len);
 574                 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl);
 575
 576                 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE;
 577                 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len);
 578         }
 579
 580         term->ctrl.mpa_len =
 581                 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE));
 582         if (qp->tx_ctx.mpa_crc_hd) {
 583                 crypto_shash_init(qp->tx_ctx.mpa_crc_hd);
 584                 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
 585                                         (u8 *)iov[0].iov_base,
 586                                         iov[0].iov_len))
 587                         goto out;
 588
 589                 if (num_frags == 3) {
 590                         if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
 591                                                 (u8 *)iov[1].iov_base,
 592                                                 iov[1].iov_len))
 593                                 goto out;
 594                 }
 595                 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc);
 596         }
 597
 598         rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate);
 599         siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
 600                    rv == len_terminate ? "success" : "failure",
 601                    __rdmap_term_layer(term), __rdmap_term_etype(term),
 602                    __rdmap_term_ecode(term), rv);
 603 out:
 604         kfree(term);
 605         kfree(err_hdr);
 606 }
 607
 608 /*
 609  * Handle all attrs other than state
 610  */
 611 static void siw_qp_modify_nonstate(struct siw_qp *qp,
 612                                    struct siw_qp_attrs *attrs,
 613                                    enum siw_qp_attr_mask mask)
 614 {
 615         if (mask & SIW_QP_ATTR_ACCESS_FLAGS) {
 616                 if (attrs->flags & SIW_RDMA_BIND_ENABLED)
 617                         qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
 618                 else
 619                         qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
 620
 621                 if (attrs->flags & SIW_RDMA_WRITE_ENABLED)
 622                         qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 623                 else
 624                         qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
 625
 626                 if (attrs->flags & SIW_RDMA_READ_ENABLED)
 627                         qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
 628                 else
 629                         qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
 630         }
 631 }
 632
 633 static int siw_qp_nextstate_from_idle(struct siw_qp *qp,
 634                                       struct siw_qp_attrs *attrs,
 635                                       enum siw_qp_attr_mask mask)
 636 {
 637         int rv = 0;
 638
 639         switch (attrs->state) {
 640         case SIW_QP_STATE_RTS:
 641                 if (attrs->flags & SIW_MPA_CRC) {
 642                         rv = siw_qp_enable_crc(qp);
 643                         if (rv)
 644                                 break;
 645                 }
 646                 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) {
 647                         siw_dbg_qp(qp, "no socket\n");
 648                         rv = -EINVAL;
 649                         break;
 650                 }
 651                 if (!(mask & SIW_QP_ATTR_MPA)) {
 652                         siw_dbg_qp(qp, "no MPA\n");
 653                         rv = -EINVAL;
 654                         break;
 655                 }
 656                 /*
 657                  * Initialize iWARP TX state
 658                  */
 659                 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
 660                 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
 661                 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
 662
 663                 /*
 664                  * Initialize iWARP RX state
 665                  */
 666                 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
 667                 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
 668                 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
 669
 670                 /*
 671                  * init IRD free queue, caller has already checked
 672                  * limits.
 673                  */
 674                 rv = siw_qp_readq_init(qp, attrs->irq_size,
 675                                        attrs->orq_size);
 676                 if (rv)
 677                         break;
 678
 679                 qp->attrs.sk = attrs->sk;
 680                 qp->attrs.state = SIW_QP_STATE_RTS;
 681
 682                 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",
 683                            attrs->flags & SIW_MPA_CRC ? "y" : "n",
 684                            qp->attrs.orq_size, qp->attrs.irq_size);
 685                 break;
 686
 687         case SIW_QP_STATE_ERROR:
 688                 siw_rq_flush(qp);
 689                 qp->attrs.state = SIW_QP_STATE_ERROR;
 690                 if (qp->cep) {
 691                         siw_cep_put(qp->cep);
 692                         qp->cep = NULL;
 693                 }
 694                 break;
 695
 696         default:
 697                 break;
 698         }
 699         return rv;
 700 }
 701
 702 static int siw_qp_nextstate_from_rts(struct siw_qp *qp,
 703                                      struct siw_qp_attrs *attrs)
 704 {
 705         int drop_conn = 0;
 706
 707         switch (attrs->state) {
 708         case SIW_QP_STATE_CLOSING:
 709                 /*
 710                  * Verbs: move to IDLE if SQ and ORQ are empty.
 711                  * Move to ERROR otherwise. But first of all we must
 712                  * close the connection. So we keep CLOSING or ERROR
 713                  * as a transient state, schedule connection drop work
 714                  * and wait for the socket state change upcall to
 715                  * come back closed.
 716                  */
 717                 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
 718                         qp->attrs.state = SIW_QP_STATE_CLOSING;
 719                 } else {
 720                         qp->attrs.state = SIW_QP_STATE_ERROR;
 721                         siw_sq_flush(qp);
 722                 }
 723                 siw_rq_flush(qp);
 724
 725                 drop_conn = 1;
 726                 break;
 727
 728         case SIW_QP_STATE_TERMINATE:
 729                 qp->attrs.state = SIW_QP_STATE_TERMINATE;
 730
 731                 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
 732                                    RDMAP_ETYPE_CATASTROPHIC,
 733                                    RDMAP_ECODE_UNSPECIFIED, 1);
 734                 drop_conn = 1;
 735                 break;
 736
 737         case SIW_QP_STATE_ERROR:
 738                 /*
 739                  * This is an emergency close.
 740                  *
 741                  * Any in progress transmit operation will get
 742                  * cancelled.
 743                  * This will likely result in a protocol failure,
 744                  * if a TX operation is in transit. The caller
 745                  * could unconditional wait to give the current
 746                  * operation a chance to complete.
 747                  * Esp., how to handle the non-empty IRQ case?
 748                  * The peer was asking for data transfer at a valid
 749                  * point in time.
 750                  */
 751                 siw_sq_flush(qp);
 752                 siw_rq_flush(qp);
 753                 qp->attrs.state = SIW_QP_STATE_ERROR;
 754                 drop_conn = 1;
 755                 break;
 756
 757         default:
 758                 break;
 759         }
 760         return drop_conn;
 761 }
 762
 763 static void siw_qp_nextstate_from_term(struct siw_qp *qp,
 764                                        struct siw_qp_attrs *attrs)
 765 {
 766         switch (attrs->state) {
 767         case SIW_QP_STATE_ERROR:
 768                 siw_rq_flush(qp);
 769                 qp->attrs.state = SIW_QP_STATE_ERROR;
 770
 771                 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
 772                         siw_sq_flush(qp);
 773                 break;
 774
 775         default:
 776                 break;
 777         }
 778 }
 779
 780 static int siw_qp_nextstate_from_close(struct siw_qp *qp,
 781                                        struct siw_qp_attrs *attrs)
 782 {
 783         int rv = 0;
 784
 785         switch (attrs->state) {
 786         case SIW_QP_STATE_IDLE:
 787                 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
 788                 qp->attrs.state = SIW_QP_STATE_IDLE;
 789                 break;
 790
 791         case SIW_QP_STATE_CLOSING:
 792                 /*
 793                  * The LLP may already moved the QP to closing
 794                  * due to graceful peer close init
 795                  */
 796                 break;
 797
 798         case SIW_QP_STATE_ERROR:
 799                 /*
 800                  * QP was moved to CLOSING by LLP event
 801                  * not yet seen by user.
 802                  */
 803                 qp->attrs.state = SIW_QP_STATE_ERROR;
 804
 805                 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
 806                         siw_sq_flush(qp);
 807
 808                 siw_rq_flush(qp);
 809                 break;
 810
 811         default:
 812                 siw_dbg_qp(qp, "state transition undefined: %s => %s\n",
 813                            siw_qp_state_to_string[qp->attrs.state],
 814                            siw_qp_state_to_string[attrs->state]);
 815
 816                 rv = -ECONNABORTED;
 817         }
 818         return rv;
 819 }
 820
 821 /*
 822  * Caller must hold qp->state_lock
 823  */
 824 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,
 825                   enum siw_qp_attr_mask mask)
 826 {
 827         int drop_conn = 0, rv = 0;
 828
 829         if (!mask)
 830                 return 0;
 831
 832         siw_dbg_qp(qp, "state: %s => %s\n",
 833                    siw_qp_state_to_string[qp->attrs.state],
 834                    siw_qp_state_to_string[attrs->state]);
 835
 836         if (mask != SIW_QP_ATTR_STATE)
 837                 siw_qp_modify_nonstate(qp, attrs, mask);
 838
 839         if (!(mask & SIW_QP_ATTR_STATE))
 840                 return 0;
 841
 842         switch (qp->attrs.state) {
 843         case SIW_QP_STATE_IDLE:
 844         case SIW_QP_STATE_RTR:
 845                 rv = siw_qp_nextstate_from_idle(qp, attrs, mask);
 846                 break;
 847
 848         case SIW_QP_STATE_RTS:
 849                 drop_conn = siw_qp_nextstate_from_rts(qp, attrs);
 850                 break;
 851
 852         case SIW_QP_STATE_TERMINATE:
 853                 siw_qp_nextstate_from_term(qp, attrs);
 854                 break;
 855
 856         case SIW_QP_STATE_CLOSING:
 857                 siw_qp_nextstate_from_close(qp, attrs);
 858                 break;
 859         default:
 860                 break;
 861         }
 862         if (drop_conn)
 863                 siw_qp_cm_drop(qp, 0);
 864
 865         return rv;
 866 }
 867
 868 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)
 869 {
 870         rreq->id = sqe->id;
 871         rreq->opcode = sqe->opcode;
 872         rreq->sge[0].laddr = sqe->sge[0].laddr;
 873         rreq->sge[0].length = sqe->sge[0].length;
 874         rreq->sge[0].lkey = sqe->sge[0].lkey;
 875         rreq->sge[1].lkey = sqe->sge[1].lkey;
 876         rreq->flags = sqe->flags | SIW_WQE_VALID;
 877         rreq->num_sge = 1;
 878 }
 879
 880 /*
 881  * Must be called with SQ locked.
 882  * To avoid complete SQ starvation by constant inbound READ requests,
 883  * the active IRQ will not be served after qp->irq_burst, if the
 884  * SQ has pending work.
 885  */
 886 int siw_activate_tx(struct siw_qp *qp)
 887 {
 888         struct siw_sqe *irqe, *sqe;
 889         struct siw_wqe *wqe = tx_wqe(qp);
 890         int rv = 1;
 891
 892         irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
 893
 894         if (irqe->flags & SIW_WQE_VALID) {
 895                 sqe = sq_get_next(qp);
 896
 897                 /*
 898                  * Avoid local WQE processing starvation in case
 899                  * of constant inbound READ request stream
 900                  */
 901                 if (sqe && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
 902                         qp->irq_burst = 0;
 903                         goto skip_irq;
 904                 }
 905                 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
 906                 wqe->wr_status = SIW_WR_QUEUED;
 907
 908                 /* start READ RESPONSE */
 909                 wqe->sqe.opcode = SIW_OP_READ_RESPONSE;
 910                 wqe->sqe.flags = 0;
 911                 if (irqe->num_sge) {
 912                         wqe->sqe.num_sge = 1;
 913                         wqe->sqe.sge[0].length = irqe->sge[0].length;
 914                         wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;
 915                         wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;
 916                 } else {
 917                         wqe->sqe.num_sge = 0;
 918                 }
 919
 920                 /* Retain original RREQ's message sequence number for
 921                  * potential error reporting cases.
 922                  */
 923                 wqe->sqe.sge[1].length = irqe->sge[1].length;
 924
 925                 wqe->sqe.rkey = irqe->rkey;
 926                 wqe->sqe.raddr = irqe->raddr;
 927
 928                 wqe->processed = 0;
 929                 qp->irq_get++;
 930
 931                 /* mark current IRQ entry free */
 932                 smp_store_mb(irqe->flags, 0);
 933
 934                 goto out;
 935         }
 936         sqe = sq_get_next(qp);
 937         if (sqe) {
 938 skip_irq:
 939                 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
 940                 wqe->wr_status = SIW_WR_QUEUED;
 941
 942                 /* First copy SQE to kernel private memory */
 943                 memcpy(&wqe->sqe, sqe, sizeof(*sqe));
 944
 945                 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {
 946                         rv = -EINVAL;
 947                         goto out;
 948                 }
 949                 if (wqe->sqe.flags & SIW_WQE_INLINE) {
 950                         if (wqe->sqe.opcode != SIW_OP_SEND &&
 951                             wqe->sqe.opcode != SIW_OP_WRITE) {
 952                                 rv = -EINVAL;
 953                                 goto out;
 954                         }
 955                         if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {
 956                                 rv = -EINVAL;
 957                                 goto out;
 958                         }
 959                         wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];
 960                         wqe->sqe.sge[0].lkey = 0;
 961                         wqe->sqe.num_sge = 1;
 962                 }
 963                 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {
 964                         /* A READ cannot be fenced */
 965                         if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||
 966                                      wqe->sqe.opcode ==
 967                                              SIW_OP_READ_LOCAL_INV)) {
 968                                 siw_dbg_qp(qp, "cannot fence read\n");
 969                                 rv = -EINVAL;
 970                                 goto out;
 971                         }
 972                         spin_lock(&qp->orq_lock);
 973
 974                         if (!siw_orq_empty(qp)) {
 975                                 qp->tx_ctx.orq_fence = 1;
 976                                 rv = 0;
 977                         }
 978                         spin_unlock(&qp->orq_lock);
 979
 980                 } else if (wqe->sqe.opcode == SIW_OP_READ ||
 981                            wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
 982                         struct siw_sqe *rreq;
 983
 984                         wqe->sqe.num_sge = 1;
 985
 986                         spin_lock(&qp->orq_lock);
 987
 988                         rreq = orq_get_free(qp);
 989                         if (rreq) {
 990                                 /*
 991                                  * Make an immediate copy in ORQ to be ready
 992                                  * to process loopback READ reply
 993                                  */
 994                                 siw_read_to_orq(rreq, &wqe->sqe);
 995                                 qp->orq_put++;
 996                         } else {
 997                                 qp->tx_ctx.orq_fence = 1;
 998                                 rv = 0;
 999                         }
1000                         spin_unlock(&qp->orq_lock);
1001                 }
1002
1003                 /* Clear SQE, can be re-used by application */
1004                 smp_store_mb(sqe->flags, 0);
1005                 qp->sq_get++;
1006         } else {
1007                 rv = 0;
1008         }
1009 out:
1010         if (unlikely(rv < 0)) {
1011                 siw_dbg_qp(qp, "error %d\n", rv);
1012                 wqe->wr_status = SIW_WR_IDLE;
1013         }
1014         return rv;
1015 }
1016
1017 /*
1018  * Check if current CQ state qualifies for calling CQ completion
1019  * handler. Must be called with CQ lock held.
1020  */
1021 static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags)
1022 {
1023         u32 cq_notify;
1024
1025         if (!cq->base_cq.comp_handler)
1026                 return false;
1027
1028         /* Read application shared notification state */
1029         cq_notify = READ_ONCE(cq->notify->flags);
1030
1031         if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) ||
1032             ((cq_notify & SIW_NOTIFY_SOLICITED) &&
1033              (flags & SIW_WQE_SOLICITED))) {
1034                 /*
1035                  * CQ notification is one-shot: Since the
1036                  * current CQE causes user notification,
1037                  * the CQ gets dis-aremd and must be re-aremd
1038                  * by the user for a new notification.
1039                  */
1040                 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT);
1041
1042                 return true;
1043         }
1044         return false;
1045 }
1046
1047 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
1048                      enum siw_wc_status status)
1049 {
1050         struct siw_cq *cq = qp->scq;
1051         int rv = 0;
1052
1053         if (cq) {
1054                 u32 sqe_flags = sqe->flags;
1055                 struct siw_cqe *cqe;
1056                 u32 idx;
1057                 unsigned long flags;
1058
1059                 spin_lock_irqsave(&cq->lock, flags);
1060
1061                 idx = cq->cq_put % cq->num_cqe;
1062                 cqe = &cq->queue[idx];
1063
1064                 if (!READ_ONCE(cqe->flags)) {
1065                         bool notify;
1066
1067                         cqe->id = sqe->id;
1068                         cqe->opcode = sqe->opcode;
1069                         cqe->status = status;
1070                         cqe->imm_data = 0;
1071                         cqe->bytes = bytes;
1072
1073                         if (cq->kernel_verbs)
1074                                 cqe->base_qp = qp->ib_qp;
1075                         else
1076                                 cqe->qp_id = qp_id(qp);
1077
1078                         /* mark CQE valid for application */
1079                         WRITE_ONCE(cqe->flags, SIW_WQE_VALID);
1080                         /* recycle SQE */
1081                         smp_store_mb(sqe->flags, 0);
1082
1083                         cq->cq_put++;
1084                         notify = siw_cq_notify_now(cq, sqe_flags);
1085
1086                         spin_unlock_irqrestore(&cq->lock, flags);
1087
1088                         if (notify) {
1089                                 siw_dbg_cq(cq, "Call completion handler\n");
1090                                 cq->base_cq.comp_handler(&cq->base_cq,
1091                                                 cq->base_cq.cq_context);
1092                         }
1093                 } else {
1094                         spin_unlock_irqrestore(&cq->lock, flags);
1095                         rv = -ENOMEM;
1096                         siw_cq_event(cq, IB_EVENT_CQ_ERR);
1097                 }
1098         } else {
1099                 /* recycle SQE */
1100                 smp_store_mb(sqe->flags, 0);
1101         }
1102         return rv;
1103 }
1104
1105 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
1106                      u32 inval_stag, enum siw_wc_status status)
1107 {
1108         struct siw_cq *cq = qp->rcq;
1109         int rv = 0;
1110
1111         if (cq) {
1112                 struct siw_cqe *cqe;
1113                 u32 idx;
1114                 unsigned long flags;
1115
1116                 spin_lock_irqsave(&cq->lock, flags);
1117
1118                 idx = cq->cq_put % cq->num_cqe;
1119                 cqe = &cq->queue[idx];
1120
1121                 if (!READ_ONCE(cqe->flags)) {
1122                         bool notify;
1123                         u8 cqe_flags = SIW_WQE_VALID;
1124
1125                         cqe->id = rqe->id;
1126                         cqe->opcode = SIW_OP_RECEIVE;
1127                         cqe->status = status;
1128                         cqe->imm_data = 0;
1129                         cqe->bytes = bytes;
1130
1131                         if (cq->kernel_verbs) {
1132                                 cqe->base_qp = qp->ib_qp;
1133                                 if (inval_stag) {
1134                                         cqe_flags |= SIW_WQE_REM_INVAL;
1135                                         cqe->inval_stag = inval_stag;
1136                                 }
1137                         } else {
1138                                 cqe->qp_id = qp_id(qp);
1139                         }
1140                         /* mark CQE valid for application */
1141                         WRITE_ONCE(cqe->flags, cqe_flags);
1142                         /* recycle RQE */
1143                         smp_store_mb(rqe->flags, 0);
1144
1145                         cq->cq_put++;
1146                         notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED);
1147
1148                         spin_unlock_irqrestore(&cq->lock, flags);
1149
1150                         if (notify) {
1151                                 siw_dbg_cq(cq, "Call completion handler\n");
1152                                 cq->base_cq.comp_handler(&cq->base_cq,
1153                                                 cq->base_cq.cq_context);
1154                         }
1155                 } else {
1156                         spin_unlock_irqrestore(&cq->lock, flags);
1157                         rv = -ENOMEM;
1158                         siw_cq_event(cq, IB_EVENT_CQ_ERR);
1159                 }
1160         } else {
1161                 /* recycle RQE */
1162                 smp_store_mb(rqe->flags, 0);
1163         }
1164         return rv;
1165 }
1166
1167 /*
1168  * siw_sq_flush()
1169  *
1170  * Flush SQ and ORRQ entries to CQ.
1171  *
1172  * Must be called with QP state write lock held.
1173  * Therefore, SQ and ORQ lock must not be taken.
1174  */
1175 void siw_sq_flush(struct siw_qp *qp)
1176 {
1177         struct siw_sqe *sqe;
1178         struct siw_wqe *wqe = tx_wqe(qp);
1179         int async_event = 0;
1180
1181         /*
1182          * Start with completing any work currently on the ORQ
1183          */
1184         while (qp->attrs.orq_size) {
1185                 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1186                 if (!READ_ONCE(sqe->flags))
1187                         break;
1188
1189                 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1190                         break;
1191
1192                 WRITE_ONCE(sqe->flags, 0);
1193                 qp->orq_get++;
1194         }
1195         /*
1196          * Flush an in-progress WQE if present
1197          */
1198         if (wqe->wr_status != SIW_WR_IDLE) {
1199                 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",
1200                            tx_type(wqe), wqe->wr_status);
1201
1202                 siw_wqe_put_mem(wqe, tx_type(wqe));
1203
1204                 if (tx_type(wqe) != SIW_OP_READ_RESPONSE &&
1205                     ((tx_type(wqe) != SIW_OP_READ &&
1206                       tx_type(wqe) != SIW_OP_READ_LOCAL_INV) ||
1207                      wqe->wr_status == SIW_WR_QUEUED))
1208                         /*
1209                          * An in-progress Read Request is already in
1210                          * the ORQ
1211                          */
1212                         siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1213                                          SIW_WC_WR_FLUSH_ERR);
1214
1215                 wqe->wr_status = SIW_WR_IDLE;
1216         }
1217         /*
1218          * Flush the Send Queue
1219          */
1220         while (qp->attrs.sq_size) {
1221                 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1222                 if (!READ_ONCE(sqe->flags))
1223                         break;
1224
1225                 async_event = 1;
1226                 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1227                         /*
1228                          * Shall IB_EVENT_SQ_DRAINED be supressed if work
1229                          * completion fails?
1230                          */
1231                         break;
1232
1233                 WRITE_ONCE(sqe->flags, 0);
1234                 qp->sq_get++;
1235         }
1236         if (async_event)
1237                 siw_qp_event(qp, IB_EVENT_SQ_DRAINED);
1238 }
1239
1240 /*
1241  * siw_rq_flush()
1242  *
1243  * Flush recv queue entries to CQ. Also
1244  * takes care of pending active tagged and untagged
1245  * inbound transfers, which have target memory
1246  * referenced.
1247  *
1248  * Must be called with QP state write lock held.
1249  * Therefore, RQ lock must not be taken.
1250  */
1251 void siw_rq_flush(struct siw_qp *qp)
1252 {
1253         struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1254
1255         /*
1256          * Flush an in-progress untagged operation if present
1257          */
1258         if (wqe->wr_status != SIW_WR_IDLE) {
1259                 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",
1260                            rx_type(wqe), wqe->wr_status);
1261
1262                 siw_wqe_put_mem(wqe, rx_type(wqe));
1263
1264                 if (rx_type(wqe) == SIW_OP_RECEIVE) {
1265                         siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1266                                          0, SIW_WC_WR_FLUSH_ERR);
1267                 } else if (rx_type(wqe) != SIW_OP_READ &&
1268                            rx_type(wqe) != SIW_OP_READ_RESPONSE &&
1269                            rx_type(wqe) != SIW_OP_WRITE) {
1270                         siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1271                 }
1272                 wqe->wr_status = SIW_WR_IDLE;
1273         }
1274         wqe = &qp->rx_tagged.wqe_active;
1275
1276         if (wqe->wr_status != SIW_WR_IDLE) {
1277                 siw_wqe_put_mem(wqe, rx_type(wqe));
1278                 wqe->wr_status = SIW_WR_IDLE;
1279         }
1280         /*
1281          * Flush the Receive Queue
1282          */
1283         while (qp->attrs.rq_size) {
1284                 struct siw_rqe *rqe =
1285                         &qp->recvq[qp->rq_get % qp->attrs.rq_size];
1286
1287                 if (!READ_ONCE(rqe->flags))
1288                         break;
1289
1290                 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1291                         break;
1292
1293                 WRITE_ONCE(rqe->flags, 0);
1294                 qp->rq_get++;
1295         }
1296 }
1297
1298 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)
1299 {
1300         int rv = xa_alloc(&sdev->qp_xa, &qp->ib_qp->qp_num, qp, xa_limit_32b,
1301                           GFP_KERNEL);
1302
1303         if (!rv) {
1304                 kref_init(&qp->ref);
1305                 qp->sdev = sdev;
1306                 qp->qp_num = qp->ib_qp->qp_num;
1307                 siw_dbg_qp(qp, "new QP\n");
1308         }
1309         return rv;
1310 }
1311
1312 void siw_free_qp(struct kref *ref)
1313 {
1314         struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);
1315         struct siw_base_qp *siw_base_qp = to_siw_base_qp(qp->ib_qp);
1316         struct siw_device *sdev = qp->sdev;
1317         unsigned long flags;
1318
1319         if (qp->cep)
1320                 siw_cep_put(qp->cep);
1321
1322         found = xa_erase(&sdev->qp_xa, qp_id(qp));
1323         WARN_ON(found != qp);
1324         spin_lock_irqsave(&sdev->lock, flags);
1325         list_del(&qp->devq);
1326         spin_unlock_irqrestore(&sdev->lock, flags);
1327
1328         vfree(qp->sendq);
1329         vfree(qp->recvq);
1330         vfree(qp->irq);
1331         vfree(qp->orq);
1332
1333         siw_put_tx_cpu(qp->tx_cpu);
1334
1335         atomic_dec(&sdev->num_qp);
1336         siw_dbg_qp(qp, "free QP\n");
1337         kfree_rcu(qp, rcu);
1338         kfree(siw_base_qp);
1339 }