drivers/infiniband/hw/hns/hns_roce_hw_v2.c

   1 /*
   2  * Copyright (c) 2016-2017 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/acpi.h>
  34 #include <linux/etherdevice.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/kernel.h>
  37 #include <linux/types.h>
  38 #include <net/addrconf.h>
  39 #include <rdma/ib_addr.h>
  40 #include <rdma/ib_cache.h>
  41 #include <rdma/ib_umem.h>
  42 #include <rdma/uverbs_ioctl.h>
  43
  44 #include "hnae3.h"
  45 #include "hns_roce_common.h"
  46 #include "hns_roce_device.h"
  47 #include "hns_roce_cmd.h"
  48 #include "hns_roce_hem.h"
  49 #include "hns_roce_hw_v2.h"
  50
  51 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
  52                             struct ib_sge *sg)
  53 {
  54         dseg->lkey = cpu_to_le32(sg->lkey);
  55         dseg->addr = cpu_to_le64(sg->addr);
  56         dseg->len  = cpu_to_le32(sg->length);
  57 }
  58
  59 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
  60                          struct hns_roce_wqe_frmr_seg *fseg,
  61                          const struct ib_reg_wr *wr)
  62 {
  63         struct hns_roce_mr *mr = to_hr_mr(wr->mr);
  64
  65         /* use ib_access_flags */
  66         roce_set_bit(rc_sq_wqe->byte_4,
  67                      V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
  68                      wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
  69         roce_set_bit(rc_sq_wqe->byte_4,
  70                      V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
  71                      wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
  72         roce_set_bit(rc_sq_wqe->byte_4,
  73                      V2_RC_FRMR_WQE_BYTE_4_RR_S,
  74                      wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
  75         roce_set_bit(rc_sq_wqe->byte_4,
  76                      V2_RC_FRMR_WQE_BYTE_4_RW_S,
  77                      wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
  78         roce_set_bit(rc_sq_wqe->byte_4,
  79                      V2_RC_FRMR_WQE_BYTE_4_LW_S,
  80                      wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
  81
  82         /* Data structure reuse may lead to confusion */
  83         rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
  84         rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
  85
  86         rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
  87         rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
  88         rc_sq_wqe->rkey = cpu_to_le32(wr->key);
  89         rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
  90
  91         fseg->pbl_size = cpu_to_le32(mr->pbl_size);
  92         roce_set_field(fseg->mode_buf_pg_sz,
  93                        V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
  94                        V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
  95                        mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
  96         roce_set_bit(fseg->mode_buf_pg_sz,
  97                      V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
  98 }
  99
 100 static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,
 101                            const struct ib_atomic_wr *wr)
 102 {
 103         if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 104                 aseg->fetchadd_swap_data = cpu_to_le64(wr->swap);
 105                 aseg->cmp_data  = cpu_to_le64(wr->compare_add);
 106         } else {
 107                 aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add);
 108                 aseg->cmp_data  = 0;
 109         }
 110 }
 111
 112 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
 113                            unsigned int *sge_ind)
 114 {
 115         struct hns_roce_v2_wqe_data_seg *dseg;
 116         struct ib_sge *sg;
 117         int num_in_wqe = 0;
 118         int extend_sge_num;
 119         int fi_sge_num;
 120         int se_sge_num;
 121         int shift;
 122         int i;
 123
 124         if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
 125                 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
 126         extend_sge_num = wr->num_sge - num_in_wqe;
 127         sg = wr->sg_list + num_in_wqe;
 128         shift = qp->hr_buf.page_shift;
 129
 130         /*
 131          * Check whether wr->num_sge sges are in the same page. If not, we
 132          * should calculate how many sges in the first page and the second
 133          * page.
 134          */
 135         dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
 136         fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
 137                       (uintptr_t)dseg) /
 138                       sizeof(struct hns_roce_v2_wqe_data_seg);
 139         if (extend_sge_num > fi_sge_num) {
 140                 se_sge_num = extend_sge_num - fi_sge_num;
 141                 for (i = 0; i < fi_sge_num; i++) {
 142                         set_data_seg_v2(dseg++, sg + i);
 143                         (*sge_ind)++;
 144                 }
 145                 dseg = get_send_extend_sge(qp,
 146                                            (*sge_ind) & (qp->sge.sge_cnt - 1));
 147                 for (i = 0; i < se_sge_num; i++) {
 148                         set_data_seg_v2(dseg++, sg + fi_sge_num + i);
 149                         (*sge_ind)++;
 150                 }
 151         } else {
 152                 for (i = 0; i < extend_sge_num; i++) {
 153                         set_data_seg_v2(dseg++, sg + i);
 154                         (*sge_ind)++;
 155                 }
 156         }
 157 }
 158
 159 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 160                              struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 161                              void *wqe, unsigned int *sge_ind,
 162                              const struct ib_send_wr **bad_wr)
 163 {
 164         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 165         struct hns_roce_v2_wqe_data_seg *dseg = wqe;
 166         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 167         int i;
 168
 169         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
 170                 if (le32_to_cpu(rc_sq_wqe->msg_len) >
 171                     hr_dev->caps.max_sq_inline) {
 172                         *bad_wr = wr;
 173                         dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
 174                                 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
 175                         return -EINVAL;
 176                 }
 177
 178                 if (wr->opcode == IB_WR_RDMA_READ) {
 179                         *bad_wr =  wr;
 180                         dev_err(hr_dev->dev, "Not support inline data!\n");
 181                         return -EINVAL;
 182                 }
 183
 184                 for (i = 0; i < wr->num_sge; i++) {
 185                         memcpy(wqe, ((void *)wr->sg_list[i].addr),
 186                                wr->sg_list[i].length);
 187                         wqe += wr->sg_list[i].length;
 188                 }
 189
 190                 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
 191                              1);
 192         } else {
 193                 if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
 194                         for (i = 0; i < wr->num_sge; i++) {
 195                                 if (likely(wr->sg_list[i].length)) {
 196                                         set_data_seg_v2(dseg, wr->sg_list + i);
 197                                         dseg++;
 198                                 }
 199                         }
 200                 } else {
 201                         roce_set_field(rc_sq_wqe->byte_20,
 202                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 203                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 204                                      (*sge_ind) & (qp->sge.sge_cnt - 1));
 205
 206                         for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
 207                                 if (likely(wr->sg_list[i].length)) {
 208                                         set_data_seg_v2(dseg, wr->sg_list + i);
 209                                         dseg++;
 210                                 }
 211                         }
 212
 213                         set_extend_sge(qp, wr, sge_ind);
 214                 }
 215
 216                 roce_set_field(rc_sq_wqe->byte_16,
 217                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 218                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
 219         }
 220
 221         return 0;
 222 }
 223
 224 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 225                                  const struct ib_qp_attr *attr,
 226                                  int attr_mask, enum ib_qp_state cur_state,
 227                                  enum ib_qp_state new_state);
 228
 229 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 230                                  const struct ib_send_wr *wr,
 231                                  const struct ib_send_wr **bad_wr)
 232 {
 233         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 234         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 235         struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
 236         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
 237         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 238         struct hns_roce_wqe_frmr_seg *fseg;
 239         struct device *dev = hr_dev->dev;
 240         struct hns_roce_v2_db sq_db;
 241         struct ib_qp_attr attr;
 242         unsigned int sge_ind = 0;
 243         unsigned int owner_bit;
 244         unsigned long flags;
 245         unsigned int ind;
 246         void *wqe = NULL;
 247         bool loopback;
 248         int attr_mask;
 249         u32 tmp_len;
 250         int ret = 0;
 251         u32 hr_op;
 252         u8 *smac;
 253         int nreq;
 254         int i;
 255
 256         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
 257                      ibqp->qp_type != IB_QPT_GSI &&
 258                      ibqp->qp_type != IB_QPT_UD)) {
 259                 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
 260                 *bad_wr = wr;
 261                 return -EOPNOTSUPP;
 262         }
 263
 264         if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
 265                      qp->state == IB_QPS_RTR)) {
 266                 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
 267                 *bad_wr = wr;
 268                 return -EINVAL;
 269         }
 270
 271         spin_lock_irqsave(&qp->sq.lock, flags);
 272         ind = qp->sq_next_wqe;
 273         sge_ind = qp->next_sge;
 274
 275         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 276                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
 277                         ret = -ENOMEM;
 278                         *bad_wr = wr;
 279                         goto out;
 280                 }
 281
 282                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
 283                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
 284                                 wr->num_sge, qp->sq.max_gs);
 285                         ret = -EINVAL;
 286                         *bad_wr = wr;
 287                         goto out;
 288                 }
 289
 290                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 291                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
 292                                                                       wr->wr_id;
 293
 294                 owner_bit =
 295                        ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
 296                 tmp_len = 0;
 297
 298                 /* Corresponding to the QP type, wqe process separately */
 299                 if (ibqp->qp_type == IB_QPT_GSI) {
 300                         ud_sq_wqe = wqe;
 301                         memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
 302
 303                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
 304                                        V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
 305                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
 306                                        V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
 307                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
 308                                        V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
 309                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
 310                                        V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
 311                         roce_set_field(ud_sq_wqe->byte_48,
 312                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
 313                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
 314                                        ah->av.mac[4]);
 315                         roce_set_field(ud_sq_wqe->byte_48,
 316                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
 317                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
 318                                        ah->av.mac[5]);
 319
 320                         /* MAC loopback */
 321                         smac = (u8 *)hr_dev->dev_addr[qp->port];
 322                         loopback = ether_addr_equal_unaligned(ah->av.mac,
 323                                                               smac) ? 1 : 0;
 324
 325                         roce_set_bit(ud_sq_wqe->byte_40,
 326                                      V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
 327
 328                         roce_set_field(ud_sq_wqe->byte_4,
 329                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
 330                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
 331                                        HNS_ROCE_V2_WQE_OP_SEND);
 332
 333                         for (i = 0; i < wr->num_sge; i++)
 334                                 tmp_len += wr->sg_list[i].length;
 335
 336                         ud_sq_wqe->msg_len =
 337                          cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
 338
 339                         switch (wr->opcode) {
 340                         case IB_WR_SEND_WITH_IMM:
 341                         case IB_WR_RDMA_WRITE_WITH_IMM:
 342                                 ud_sq_wqe->immtdata =
 343                                       cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 344                                 break;
 345                         default:
 346                                 ud_sq_wqe->immtdata = 0;
 347                                 break;
 348                         }
 349
 350                         /* Set sig attr */
 351                         roce_set_bit(ud_sq_wqe->byte_4,
 352                                    V2_UD_SEND_WQE_BYTE_4_CQE_S,
 353                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 354
 355                         /* Set se attr */
 356                         roce_set_bit(ud_sq_wqe->byte_4,
 357                                   V2_UD_SEND_WQE_BYTE_4_SE_S,
 358                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 359
 360                         roce_set_bit(ud_sq_wqe->byte_4,
 361                                      V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
 362
 363                         roce_set_field(ud_sq_wqe->byte_16,
 364                                        V2_UD_SEND_WQE_BYTE_16_PD_M,
 365                                        V2_UD_SEND_WQE_BYTE_16_PD_S,
 366                                        to_hr_pd(ibqp->pd)->pdn);
 367
 368                         roce_set_field(ud_sq_wqe->byte_16,
 369                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
 370                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
 371                                        wr->num_sge);
 372
 373                         roce_set_field(ud_sq_wqe->byte_20,
 374                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 375                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 376                                      sge_ind & (qp->sge.sge_cnt - 1));
 377
 378                         roce_set_field(ud_sq_wqe->byte_24,
 379                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
 380                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
 381                         ud_sq_wqe->qkey =
 382                              cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
 383                              qp->qkey : ud_wr(wr)->remote_qkey);
 384                         roce_set_field(ud_sq_wqe->byte_32,
 385                                        V2_UD_SEND_WQE_BYTE_32_DQPN_M,
 386                                        V2_UD_SEND_WQE_BYTE_32_DQPN_S,
 387                                        ud_wr(wr)->remote_qpn);
 388
 389                         roce_set_field(ud_sq_wqe->byte_36,
 390                                        V2_UD_SEND_WQE_BYTE_36_VLAN_M,
 391                                        V2_UD_SEND_WQE_BYTE_36_VLAN_S,
 392                                        le16_to_cpu(ah->av.vlan));
 393                         roce_set_field(ud_sq_wqe->byte_36,
 394                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
 395                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
 396                                        ah->av.hop_limit);
 397                         roce_set_field(ud_sq_wqe->byte_36,
 398                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
 399                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
 400                                        ah->av.sl_tclass_flowlabel >>
 401                                        HNS_ROCE_TCLASS_SHIFT);
 402                         roce_set_field(ud_sq_wqe->byte_40,
 403                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
 404                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
 405                                        ah->av.sl_tclass_flowlabel &
 406                                        HNS_ROCE_FLOW_LABEL_MASK);
 407                         roce_set_field(ud_sq_wqe->byte_40,
 408                                        V2_UD_SEND_WQE_BYTE_40_SL_M,
 409                                        V2_UD_SEND_WQE_BYTE_40_SL_S,
 410                                       le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
 411                                       HNS_ROCE_SL_SHIFT);
 412                         roce_set_field(ud_sq_wqe->byte_40,
 413                                        V2_UD_SEND_WQE_BYTE_40_PORTN_M,
 414                                        V2_UD_SEND_WQE_BYTE_40_PORTN_S,
 415                                        qp->port);
 416
 417                         roce_set_bit(ud_sq_wqe->byte_40,
 418                                      V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
 419                                      ah->av.vlan_en ? 1 : 0);
 420                         roce_set_field(ud_sq_wqe->byte_48,
 421                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
 422                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
 423                                        hns_get_gid_index(hr_dev, qp->phy_port,
 424                                                          ah->av.gid_index));
 425
 426                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
 427                                GID_LEN_V2);
 428
 429                         set_extend_sge(qp, wr, &sge_ind);
 430                         ind++;
 431                 } else if (ibqp->qp_type == IB_QPT_RC) {
 432                         rc_sq_wqe = wqe;
 433                         memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
 434                         for (i = 0; i < wr->num_sge; i++)
 435                                 tmp_len += wr->sg_list[i].length;
 436
 437                         rc_sq_wqe->msg_len =
 438                          cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
 439
 440                         switch (wr->opcode) {
 441                         case IB_WR_SEND_WITH_IMM:
 442                         case IB_WR_RDMA_WRITE_WITH_IMM:
 443                                 rc_sq_wqe->immtdata =
 444                                       cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 445                                 break;
 446                         case IB_WR_SEND_WITH_INV:
 447                                 rc_sq_wqe->inv_key =
 448                                         cpu_to_le32(wr->ex.invalidate_rkey);
 449                                 break;
 450                         default:
 451                                 rc_sq_wqe->immtdata = 0;
 452                                 break;
 453                         }
 454
 455                         roce_set_bit(rc_sq_wqe->byte_4,
 456                                      V2_RC_SEND_WQE_BYTE_4_FENCE_S,
 457                                      (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
 458
 459                         roce_set_bit(rc_sq_wqe->byte_4,
 460                                   V2_RC_SEND_WQE_BYTE_4_SE_S,
 461                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 462
 463                         roce_set_bit(rc_sq_wqe->byte_4,
 464                                    V2_RC_SEND_WQE_BYTE_4_CQE_S,
 465                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 466
 467                         roce_set_bit(rc_sq_wqe->byte_4,
 468                                      V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
 469
 470                         wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
 471                         switch (wr->opcode) {
 472                         case IB_WR_RDMA_READ:
 473                                 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;
 474                                 rc_sq_wqe->rkey =
 475                                         cpu_to_le32(rdma_wr(wr)->rkey);
 476                                 rc_sq_wqe->va =
 477                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
 478                                 break;
 479                         case IB_WR_RDMA_WRITE:
 480                                 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;
 481                                 rc_sq_wqe->rkey =
 482                                         cpu_to_le32(rdma_wr(wr)->rkey);
 483                                 rc_sq_wqe->va =
 484                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
 485                                 break;
 486                         case IB_WR_RDMA_WRITE_WITH_IMM:
 487                                 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;
 488                                 rc_sq_wqe->rkey =
 489                                         cpu_to_le32(rdma_wr(wr)->rkey);
 490                                 rc_sq_wqe->va =
 491                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
 492                                 break;
 493                         case IB_WR_SEND:
 494                                 hr_op = HNS_ROCE_V2_WQE_OP_SEND;
 495                                 break;
 496                         case IB_WR_SEND_WITH_INV:
 497                                 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;
 498                                 break;
 499                         case IB_WR_SEND_WITH_IMM:
 500                                 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;
 501                                 break;
 502                         case IB_WR_LOCAL_INV:
 503                                 hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;
 504                                 roce_set_bit(rc_sq_wqe->byte_4,
 505                                                V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
 506                                 rc_sq_wqe->inv_key =
 507                                             cpu_to_le32(wr->ex.invalidate_rkey);
 508                                 break;
 509                         case IB_WR_REG_MR:
 510                                 hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;
 511                                 fseg = wqe;
 512                                 set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));
 513                                 break;
 514                         case IB_WR_ATOMIC_CMP_AND_SWP:
 515                                 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;
 516                                 rc_sq_wqe->rkey =
 517                                         cpu_to_le32(atomic_wr(wr)->rkey);
 518                                 rc_sq_wqe->va =
 519                                         cpu_to_le64(atomic_wr(wr)->remote_addr);
 520                                 break;
 521                         case IB_WR_ATOMIC_FETCH_AND_ADD:
 522                                 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;
 523                                 rc_sq_wqe->rkey =
 524                                         cpu_to_le32(atomic_wr(wr)->rkey);
 525                                 rc_sq_wqe->va =
 526                                         cpu_to_le64(atomic_wr(wr)->remote_addr);
 527                                 break;
 528                         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
 529                                 hr_op =
 530                                        HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;
 531                                 break;
 532                         case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
 533                                 hr_op =
 534                                       HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;
 535                                 break;
 536                         default:
 537                                 hr_op = HNS_ROCE_V2_WQE_OP_MASK;
 538                                 break;
 539                         }
 540
 541                         roce_set_field(rc_sq_wqe->byte_4,
 542                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 543                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);
 544
 545                         if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
 546                             wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
 547                                 struct hns_roce_v2_wqe_data_seg *dseg;
 548
 549                                 dseg = wqe;
 550                                 set_data_seg_v2(dseg, wr->sg_list);
 551                                 wqe += sizeof(struct hns_roce_v2_wqe_data_seg);
 552                                 set_atomic_seg(wqe, atomic_wr(wr));
 553                                 roce_set_field(rc_sq_wqe->byte_16,
 554                                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 555                                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
 556                                                wr->num_sge);
 557                         } else if (wr->opcode != IB_WR_REG_MR) {
 558                                 ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,
 559                                                         wqe, &sge_ind, bad_wr);
 560                                 if (ret)
 561                                         goto out;
 562                         }
 563
 564                         ind++;
 565                 } else {
 566                         dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
 567                         spin_unlock_irqrestore(&qp->sq.lock, flags);
 568                         *bad_wr = wr;
 569                         return -EOPNOTSUPP;
 570                 }
 571         }
 572
 573 out:
 574         if (likely(nreq)) {
 575                 qp->sq.head += nreq;
 576                 /* Memory barrier */
 577                 wmb();
 578
 579                 sq_db.byte_4 = 0;
 580                 sq_db.parameter = 0;
 581
 582                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
 583                                V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
 584                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
 585                                V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
 586                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
 587                                V2_DB_PARAMETER_IDX_S,
 588                                qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
 589                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
 590                                V2_DB_PARAMETER_SL_S, qp->sl);
 591
 592                 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg_l);
 593
 594                 qp->sq_next_wqe = ind;
 595                 qp->next_sge = sge_ind;
 596
 597                 if (qp->state == IB_QPS_ERR) {
 598                         attr_mask = IB_QP_STATE;
 599                         attr.qp_state = IB_QPS_ERR;
 600
 601                         ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
 602                                                     qp->state, IB_QPS_ERR);
 603                         if (ret) {
 604                                 spin_unlock_irqrestore(&qp->sq.lock, flags);
 605                                 *bad_wr = wr;
 606                                 return ret;
 607                         }
 608                 }
 609         }
 610
 611         spin_unlock_irqrestore(&qp->sq.lock, flags);
 612
 613         return ret;
 614 }
 615
 616 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 617                                  const struct ib_recv_wr *wr,
 618                                  const struct ib_recv_wr **bad_wr)
 619 {
 620         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 621         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 622         struct hns_roce_v2_wqe_data_seg *dseg;
 623         struct hns_roce_rinl_sge *sge_list;
 624         struct device *dev = hr_dev->dev;
 625         struct ib_qp_attr attr;
 626         unsigned long flags;
 627         void *wqe = NULL;
 628         int attr_mask;
 629         int ret = 0;
 630         int nreq;
 631         int ind;
 632         int i;
 633
 634         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 635         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
 636
 637         if (hr_qp->state == IB_QPS_RESET) {
 638                 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 639                 *bad_wr = wr;
 640                 return -EINVAL;
 641         }
 642
 643         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 644                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
 645                         hr_qp->ibqp.recv_cq)) {
 646                         ret = -ENOMEM;
 647                         *bad_wr = wr;
 648                         goto out;
 649                 }
 650
 651                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
 652                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
 653                                 wr->num_sge, hr_qp->rq.max_gs);
 654                         ret = -EINVAL;
 655                         *bad_wr = wr;
 656                         goto out;
 657                 }
 658
 659                 wqe = get_recv_wqe(hr_qp, ind);
 660                 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
 661                 for (i = 0; i < wr->num_sge; i++) {
 662                         if (!wr->sg_list[i].length)
 663                                 continue;
 664                         set_data_seg_v2(dseg, wr->sg_list + i);
 665                         dseg++;
 666                 }
 667
 668                 if (i < hr_qp->rq.max_gs) {
 669                         dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 670                         dseg->addr = 0;
 671                 }
 672
 673                 /* rq support inline data */
 674                 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
 675                         sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
 676                         hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
 677                                                                (u32)wr->num_sge;
 678                         for (i = 0; i < wr->num_sge; i++) {
 679                                 sge_list[i].addr =
 680                                                (void *)(u64)wr->sg_list[i].addr;
 681                                 sge_list[i].len = wr->sg_list[i].length;
 682                         }
 683                 }
 684
 685                 hr_qp->rq.wrid[ind] = wr->wr_id;
 686
 687                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
 688         }
 689
 690 out:
 691         if (likely(nreq)) {
 692                 hr_qp->rq.head += nreq;
 693                 /* Memory barrier */
 694                 wmb();
 695
 696                 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
 697
 698                 if (hr_qp->state == IB_QPS_ERR) {
 699                         attr_mask = IB_QP_STATE;
 700                         attr.qp_state = IB_QPS_ERR;
 701
 702                         ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
 703                                                     attr_mask, hr_qp->state,
 704                                                     IB_QPS_ERR);
 705                         if (ret) {
 706                                 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 707                                 *bad_wr = wr;
 708                                 return ret;
 709                         }
 710                 }
 711         }
 712         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 713
 714         return ret;
 715 }
 716
 717 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
 718                                       unsigned long instance_stage,
 719                                       unsigned long reset_stage)
 720 {
 721         /* When hardware reset has been completed once or more, we should stop
 722          * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
 723          * function, we should exit with error. If now at HNAE3_INIT_CLIENT
 724          * stage of soft reset process, we should exit with error, and then
 725          * HNAE3_INIT_CLIENT related process can rollback the operation like
 726          * notifing hardware to free resources, HNAE3_INIT_CLIENT related
 727          * process will exit with error to notify NIC driver to reschedule soft
 728          * reset process once again.
 729          */
 730         hr_dev->is_reset = true;
 731         hr_dev->dis_db = true;
 732
 733         if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
 734             instance_stage == HNS_ROCE_STATE_INIT)
 735                 return CMD_RST_PRC_EBUSY;
 736
 737         return CMD_RST_PRC_SUCCESS;
 738 }
 739
 740 static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
 741                                         unsigned long instance_stage,
 742                                         unsigned long reset_stage)
 743 {
 744         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 745         struct hnae3_handle *handle = priv->handle;
 746         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 747
 748         /* When hardware reset is detected, we should stop sending mailbox&cmq&
 749          * doorbell to hardware. If now in .init_instance() function, we should
 750          * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
 751          * process, we should exit with error, and then HNAE3_INIT_CLIENT
 752          * related process can rollback the operation like notifing hardware to
 753          * free resources, HNAE3_INIT_CLIENT related process will exit with
 754          * error to notify NIC driver to reschedule soft reset process once
 755          * again.
 756          */
 757         hr_dev->dis_db = true;
 758         if (!ops->get_hw_reset_stat(handle))
 759                 hr_dev->is_reset = true;
 760
 761         if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
 762             instance_stage == HNS_ROCE_STATE_INIT)
 763                 return CMD_RST_PRC_EBUSY;
 764
 765         return CMD_RST_PRC_SUCCESS;
 766 }
 767
 768 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
 769 {
 770         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 771         struct hnae3_handle *handle = priv->handle;
 772         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 773
 774         /* When software reset is detected at .init_instance() function, we
 775          * should stop sending mailbox&cmq&doorbell to hardware, and exit
 776          * with error.
 777          */
 778         hr_dev->dis_db = true;
 779         if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
 780                 hr_dev->is_reset = true;
 781
 782         return CMD_RST_PRC_EBUSY;
 783 }
 784
 785 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
 786 {
 787         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 788         struct hnae3_handle *handle = priv->handle;
 789         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 790         unsigned long instance_stage;   /* the current instance stage */
 791         unsigned long reset_stage;      /* the current reset stage */
 792         unsigned long reset_cnt;
 793         bool sw_resetting;
 794         bool hw_resetting;
 795
 796         if (hr_dev->is_reset)
 797                 return CMD_RST_PRC_SUCCESS;
 798
 799         /* Get information about reset from NIC driver or RoCE driver itself,
 800          * the meaning of the following variables from NIC driver are described
 801          * as below:
 802          * reset_cnt -- The count value of completed hardware reset.
 803          * hw_resetting -- Whether hardware device is resetting now.
 804          * sw_resetting -- Whether NIC's software reset process is running now.
 805          */
 806         instance_stage = handle->rinfo.instance_state;
 807         reset_stage = handle->rinfo.reset_state;
 808         reset_cnt = ops->ae_dev_reset_cnt(handle);
 809         hw_resetting = ops->get_hw_reset_stat(handle);
 810         sw_resetting = ops->ae_dev_resetting(handle);
 811
 812         if (reset_cnt != hr_dev->reset_cnt)
 813                 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
 814                                                   reset_stage);
 815         else if (hw_resetting)
 816                 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
 817                                                     reset_stage);
 818         else if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
 819                 return hns_roce_v2_cmd_sw_resetting(hr_dev);
 820
 821         return 0;
 822 }
 823
 824 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
 825 {
 826         int ntu = ring->next_to_use;
 827         int ntc = ring->next_to_clean;
 828         int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
 829
 830         return ring->desc_num - used - 1;
 831 }
 832
 833 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
 834                                    struct hns_roce_v2_cmq_ring *ring)
 835 {
 836         int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
 837
 838         ring->desc = kzalloc(size, GFP_KERNEL);
 839         if (!ring->desc)
 840                 return -ENOMEM;
 841
 842         ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
 843                                              DMA_BIDIRECTIONAL);
 844         if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
 845                 ring->desc_dma_addr = 0;
 846                 kfree(ring->desc);
 847                 ring->desc = NULL;
 848                 return -ENOMEM;
 849         }
 850
 851         return 0;
 852 }
 853
 854 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
 855                                    struct hns_roce_v2_cmq_ring *ring)
 856 {
 857         dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
 858                          ring->desc_num * sizeof(struct hns_roce_cmq_desc),
 859                          DMA_BIDIRECTIONAL);
 860
 861         ring->desc_dma_addr = 0;
 862         kfree(ring->desc);
 863 }
 864
 865 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
 866 {
 867         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 868         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
 869                                             &priv->cmq.csq : &priv->cmq.crq;
 870
 871         ring->flag = ring_type;
 872         ring->next_to_clean = 0;
 873         ring->next_to_use = 0;
 874
 875         return hns_roce_alloc_cmq_desc(hr_dev, ring);
 876 }
 877
 878 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
 879 {
 880         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 881         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
 882                                             &priv->cmq.csq : &priv->cmq.crq;
 883         dma_addr_t dma = ring->desc_dma_addr;
 884
 885         if (ring_type == TYPE_CSQ) {
 886                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
 887                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
 888                            upper_32_bits(dma));
 889                 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
 890                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
 891                            HNS_ROCE_CMQ_ENABLE);
 892                 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
 893                 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
 894         } else {
 895                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
 896                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
 897                            upper_32_bits(dma));
 898                 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
 899                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
 900                            HNS_ROCE_CMQ_ENABLE);
 901                 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
 902                 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
 903         }
 904 }
 905
 906 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
 907 {
 908         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 909         int ret;
 910
 911         /* Setup the queue entries for command queue */
 912         priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
 913         priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
 914
 915         /* Setup the lock for command queue */
 916         spin_lock_init(&priv->cmq.csq.lock);
 917         spin_lock_init(&priv->cmq.crq.lock);
 918
 919         /* Setup Tx write back timeout */
 920         priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
 921
 922         /* Init CSQ */
 923         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
 924         if (ret) {
 925                 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
 926                 return ret;
 927         }
 928
 929         /* Init CRQ */
 930         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
 931         if (ret) {
 932                 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
 933                 goto err_crq;
 934         }
 935
 936         /* Init CSQ REG */
 937         hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
 938
 939         /* Init CRQ REG */
 940         hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
 941
 942         return 0;
 943
 944 err_crq:
 945         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
 946
 947         return ret;
 948 }
 949
 950 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
 951 {
 952         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 953
 954         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
 955         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
 956 }
 957
 958 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
 959                                           enum hns_roce_opcode_type opcode,
 960                                           bool is_read)
 961 {
 962         memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
 963         desc->opcode = cpu_to_le16(opcode);
 964         desc->flag =
 965                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
 966         if (is_read)
 967                 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
 968         else
 969                 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
 970 }
 971
 972 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
 973 {
 974         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 975         u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
 976
 977         return head == priv->cmq.csq.next_to_use;
 978 }
 979
 980 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
 981 {
 982         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 983         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
 984         struct hns_roce_cmq_desc *desc;
 985         u16 ntc = csq->next_to_clean;
 986         u32 head;
 987         int clean = 0;
 988
 989         desc = &csq->desc[ntc];
 990         head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
 991         while (head != ntc) {
 992                 memset(desc, 0, sizeof(*desc));
 993                 ntc++;
 994                 if (ntc == csq->desc_num)
 995                         ntc = 0;
 996                 desc = &csq->desc[ntc];
 997                 clean++;
 998         }
 999         csq->next_to_clean = ntc;
1000
1001         return clean;
1002 }
1003
1004 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1005                                struct hns_roce_cmq_desc *desc, int num)
1006 {
1007         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
1008         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1009         struct hns_roce_cmq_desc *desc_to_use;
1010         bool complete = false;
1011         u32 timeout = 0;
1012         int handle = 0;
1013         u16 desc_ret;
1014         int ret = 0;
1015         int ntc;
1016
1017         spin_lock_bh(&csq->lock);
1018
1019         if (num > hns_roce_cmq_space(csq)) {
1020                 spin_unlock_bh(&csq->lock);
1021                 return -EBUSY;
1022         }
1023
1024         /*
1025          * Record the location of desc in the cmq for this time
1026          * which will be use for hardware to write back
1027          */
1028         ntc = csq->next_to_use;
1029
1030         while (handle < num) {
1031                 desc_to_use = &csq->desc[csq->next_to_use];
1032                 *desc_to_use = desc[handle];
1033                 dev_dbg(hr_dev->dev, "set cmq desc:\n");
1034                 csq->next_to_use++;
1035                 if (csq->next_to_use == csq->desc_num)
1036                         csq->next_to_use = 0;
1037                 handle++;
1038         }
1039
1040         /* Write to hardware */
1041         roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
1042
1043         /*
1044          * If the command is sync, wait for the firmware to write back,
1045          * if multi descriptors to be sent, use the first one to check
1046          */
1047         if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
1048                 do {
1049                         if (hns_roce_cmq_csq_done(hr_dev))
1050                                 break;
1051                         udelay(1);
1052                         timeout++;
1053                 } while (timeout < priv->cmq.tx_timeout);
1054         }
1055
1056         if (hns_roce_cmq_csq_done(hr_dev)) {
1057                 complete = true;
1058                 handle = 0;
1059                 while (handle < num) {
1060                         /* get the result of hardware write back */
1061                         desc_to_use = &csq->desc[ntc];
1062                         desc[handle] = *desc_to_use;
1063                         dev_dbg(hr_dev->dev, "Get cmq desc:\n");
1064                         desc_ret = desc[handle].retval;
1065                         if (desc_ret == CMD_EXEC_SUCCESS)
1066                                 ret = 0;
1067                         else
1068                                 ret = -EIO;
1069                         priv->cmq.last_status = desc_ret;
1070                         ntc++;
1071                         handle++;
1072                         if (ntc == csq->desc_num)
1073                                 ntc = 0;
1074                 }
1075         }
1076
1077         if (!complete)
1078                 ret = -EAGAIN;
1079
1080         /* clean the command send queue */
1081         handle = hns_roce_cmq_csq_clean(hr_dev);
1082         if (handle != num)
1083                 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
1084                          handle, num);
1085
1086         spin_unlock_bh(&csq->lock);
1087
1088         return ret;
1089 }
1090
1091 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1092                              struct hns_roce_cmq_desc *desc, int num)
1093 {
1094         int retval;
1095         int ret;
1096
1097         ret = hns_roce_v2_rst_process_cmd(hr_dev);
1098         if (ret == CMD_RST_PRC_SUCCESS)
1099                 return 0;
1100         if (ret == CMD_RST_PRC_EBUSY)
1101                 return -EBUSY;
1102
1103         ret = __hns_roce_cmq_send(hr_dev, desc, num);
1104         if (ret) {
1105                 retval = hns_roce_v2_rst_process_cmd(hr_dev);
1106                 if (retval == CMD_RST_PRC_SUCCESS)
1107                         return 0;
1108                 else if (retval == CMD_RST_PRC_EBUSY)
1109                         return -EBUSY;
1110         }
1111
1112         return ret;
1113 }
1114
1115 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1116 {
1117         struct hns_roce_query_version *resp;
1118         struct hns_roce_cmq_desc desc;
1119         int ret;
1120
1121         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1122         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1123         if (ret)
1124                 return ret;
1125
1126         resp = (struct hns_roce_query_version *)desc.data;
1127         hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
1128         hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1129
1130         return 0;
1131 }
1132
1133 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1134 {
1135         struct hns_roce_func_clear *resp;
1136         struct hns_roce_cmq_desc desc;
1137         unsigned long end;
1138         int ret;
1139
1140         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1141         resp = (struct hns_roce_func_clear *)desc.data;
1142
1143         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1144         if (ret) {
1145                 dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n",
1146                          ret);
1147                 return;
1148         }
1149
1150         msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1151         end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1152         while (end) {
1153                 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1154                 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1155
1156                 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1157                                               true);
1158
1159                 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1160                 if (ret)
1161                         continue;
1162
1163                 if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) {
1164                         hr_dev->is_reset = true;
1165                         return;
1166                 }
1167         }
1168
1169         dev_err(hr_dev->dev, "Func clear fail.\n");
1170 }
1171
1172 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1173 {
1174         struct hns_roce_query_fw_info *resp;
1175         struct hns_roce_cmq_desc desc;
1176         int ret;
1177
1178         hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1179         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1180         if (ret)
1181                 return ret;
1182
1183         resp = (struct hns_roce_query_fw_info *)desc.data;
1184         hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1185
1186         return 0;
1187 }
1188
1189 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1190 {
1191         struct hns_roce_cfg_global_param *req;
1192         struct hns_roce_cmq_desc desc;
1193
1194         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1195                                       false);
1196
1197         req = (struct hns_roce_cfg_global_param *)desc.data;
1198         memset(req, 0, sizeof(*req));
1199         roce_set_field(req->time_cfg_udp_port,
1200                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1201                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1202         roce_set_field(req->time_cfg_udp_port,
1203                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1204                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
1205
1206         return hns_roce_cmq_send(hr_dev, &desc, 1);
1207 }
1208
1209 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1210 {
1211         struct hns_roce_cmq_desc desc[2];
1212         struct hns_roce_pf_res_a *req_a;
1213         struct hns_roce_pf_res_b *req_b;
1214         int ret;
1215         int i;
1216
1217         for (i = 0; i < 2; i++) {
1218                 hns_roce_cmq_setup_basic_desc(&desc[i],
1219                                               HNS_ROCE_OPC_QUERY_PF_RES, true);
1220
1221                 if (i == 0)
1222                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1223                 else
1224                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1225         }
1226
1227         ret = hns_roce_cmq_send(hr_dev, desc, 2);
1228         if (ret)
1229                 return ret;
1230
1231         req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1232         req_b = (struct hns_roce_pf_res_b *)desc[1].data;
1233
1234         hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
1235                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1236                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_S);
1237         hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
1238                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1239                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
1240         hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
1241                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1242                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_S);
1243         hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
1244                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1245                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1246
1247         hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1248                                              PF_RES_DATA_3_PF_SL_NUM_M,
1249                                              PF_RES_DATA_3_PF_SL_NUM_S);
1250         hr_dev->caps.sccc_bt_num = roce_get_field(req_b->sccc_bt_idx_num,
1251                                              PF_RES_DATA_4_PF_SCCC_BT_NUM_M,
1252                                              PF_RES_DATA_4_PF_SCCC_BT_NUM_S);
1253
1254         return 0;
1255 }
1256
1257 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
1258 {
1259         struct hns_roce_pf_timer_res_a *req_a;
1260         struct hns_roce_cmq_desc desc[2];
1261         int ret, i;
1262
1263         for (i = 0; i < 2; i++) {
1264                 hns_roce_cmq_setup_basic_desc(&desc[i],
1265                                               HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1266                                               true);
1267
1268                 if (i == 0)
1269                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1270                 else
1271                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1272         }
1273
1274         ret = hns_roce_cmq_send(hr_dev, desc, 2);
1275         if (ret)
1276                 return ret;
1277
1278         req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;
1279
1280         hr_dev->caps.qpc_timer_bt_num =
1281                                 roce_get_field(req_a->qpc_timer_bt_idx_num,
1282                                         PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
1283                                         PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
1284         hr_dev->caps.cqc_timer_bt_num =
1285                                 roce_get_field(req_a->cqc_timer_bt_idx_num,
1286                                         PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
1287                                         PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
1288
1289         return 0;
1290 }
1291
1292 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1293                                                   int vf_id)
1294 {
1295         struct hns_roce_cmq_desc desc;
1296         struct hns_roce_vf_switch *swt;
1297         int ret;
1298
1299         swt = (struct hns_roce_vf_switch *)desc.data;
1300         hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1301         swt->rocee_sel |= cpu_to_le16(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1302         roce_set_field(swt->fun_id,
1303                         VF_SWITCH_DATA_FUN_ID_VF_ID_M,
1304                         VF_SWITCH_DATA_FUN_ID_VF_ID_S,
1305                         vf_id);
1306         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1307         if (ret)
1308                 return ret;
1309         desc.flag =
1310                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
1311         desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1312         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1);
1313         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 1);
1314         roce_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1);
1315
1316         return hns_roce_cmq_send(hr_dev, &desc, 1);
1317 }
1318
1319 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1320 {
1321         struct hns_roce_cmq_desc desc[2];
1322         struct hns_roce_vf_res_a *req_a;
1323         struct hns_roce_vf_res_b *req_b;
1324         int i;
1325
1326         req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1327         req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1328         memset(req_a, 0, sizeof(*req_a));
1329         memset(req_b, 0, sizeof(*req_b));
1330         for (i = 0; i < 2; i++) {
1331                 hns_roce_cmq_setup_basic_desc(&desc[i],
1332                                               HNS_ROCE_OPC_ALLOC_VF_RES, false);
1333
1334                 if (i == 0)
1335                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1336                 else
1337                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1338
1339                 if (i == 0) {
1340                         roce_set_field(req_a->vf_qpc_bt_idx_num,
1341                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1342                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1343                         roce_set_field(req_a->vf_qpc_bt_idx_num,
1344                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1345                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1346                                        HNS_ROCE_VF_QPC_BT_NUM);
1347
1348                         roce_set_field(req_a->vf_srqc_bt_idx_num,
1349                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1350                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1351                         roce_set_field(req_a->vf_srqc_bt_idx_num,
1352                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1353                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1354                                        HNS_ROCE_VF_SRQC_BT_NUM);
1355
1356                         roce_set_field(req_a->vf_cqc_bt_idx_num,
1357                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1358                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1359                         roce_set_field(req_a->vf_cqc_bt_idx_num,
1360                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1361                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1362                                        HNS_ROCE_VF_CQC_BT_NUM);
1363
1364                         roce_set_field(req_a->vf_mpt_bt_idx_num,
1365                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1366                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1367                         roce_set_field(req_a->vf_mpt_bt_idx_num,
1368                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1369                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1370                                        HNS_ROCE_VF_MPT_BT_NUM);
1371
1372                         roce_set_field(req_a->vf_eqc_bt_idx_num,
1373                                        VF_RES_A_DATA_5_VF_EQC_IDX_M,
1374                                        VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1375                         roce_set_field(req_a->vf_eqc_bt_idx_num,
1376                                        VF_RES_A_DATA_5_VF_EQC_NUM_M,
1377                                        VF_RES_A_DATA_5_VF_EQC_NUM_S,
1378                                        HNS_ROCE_VF_EQC_NUM);
1379                 } else {
1380                         roce_set_field(req_b->vf_smac_idx_num,
1381                                        VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1382                                        VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1383                         roce_set_field(req_b->vf_smac_idx_num,
1384                                        VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1385                                        VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1386                                        HNS_ROCE_VF_SMAC_NUM);
1387
1388                         roce_set_field(req_b->vf_sgid_idx_num,
1389                                        VF_RES_B_DATA_2_VF_SGID_IDX_M,
1390                                        VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1391                         roce_set_field(req_b->vf_sgid_idx_num,
1392                                        VF_RES_B_DATA_2_VF_SGID_NUM_M,
1393                                        VF_RES_B_DATA_2_VF_SGID_NUM_S,
1394                                        HNS_ROCE_VF_SGID_NUM);
1395
1396                         roce_set_field(req_b->vf_qid_idx_sl_num,
1397                                        VF_RES_B_DATA_3_VF_QID_IDX_M,
1398                                        VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1399                         roce_set_field(req_b->vf_qid_idx_sl_num,
1400                                        VF_RES_B_DATA_3_VF_SL_NUM_M,
1401                                        VF_RES_B_DATA_3_VF_SL_NUM_S,
1402                                        HNS_ROCE_VF_SL_NUM);
1403
1404                         roce_set_field(req_b->vf_sccc_idx_num,
1405                                        VF_RES_B_DATA_4_VF_SCCC_BT_IDX_M,
1406                                        VF_RES_B_DATA_4_VF_SCCC_BT_IDX_S, 0);
1407                         roce_set_field(req_b->vf_sccc_idx_num,
1408                                        VF_RES_B_DATA_4_VF_SCCC_BT_NUM_M,
1409                                        VF_RES_B_DATA_4_VF_SCCC_BT_NUM_S,
1410                                        HNS_ROCE_VF_SCCC_BT_NUM);
1411                 }
1412         }
1413
1414         return hns_roce_cmq_send(hr_dev, desc, 2);
1415 }
1416
1417 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1418 {
1419         u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1420         u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1421         u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1422         u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1423         u8 sccc_hop_num = hr_dev->caps.sccc_hop_num;
1424         struct hns_roce_cfg_bt_attr *req;
1425         struct hns_roce_cmq_desc desc;
1426
1427         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1428         req = (struct hns_roce_cfg_bt_attr *)desc.data;
1429         memset(req, 0, sizeof(*req));
1430
1431         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1432                        CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1433                        hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1434         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1435                        CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1436                        hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1437         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1438                        CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1439                        qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1440
1441         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1442                        CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1443                        hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1444         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1445                        CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1446                        hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1447         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1448                        CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1449                        srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1450
1451         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1452                        CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1453                        hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1454         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1455                        CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1456                        hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1457         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1458                        CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1459                        cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1460
1461         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1462                        CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1463                        hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1464         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1465                        CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1466                        hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1467         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1468                        CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1469                        mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1470
1471         roce_set_field(req->vf_sccc_cfg,
1472                        CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_M,
1473                        CFG_BT_ATTR_DATA_4_VF_SCCC_BA_PGSZ_S,
1474                        hr_dev->caps.sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1475         roce_set_field(req->vf_sccc_cfg,
1476                        CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_M,
1477                        CFG_BT_ATTR_DATA_4_VF_SCCC_BUF_PGSZ_S,
1478                        hr_dev->caps.sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1479         roce_set_field(req->vf_sccc_cfg,
1480                        CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_M,
1481                        CFG_BT_ATTR_DATA_4_VF_SCCC_HOPNUM_S,
1482                        sccc_hop_num ==
1483                               HNS_ROCE_HOP_NUM_0 ? 0 : sccc_hop_num);
1484
1485         return hns_roce_cmq_send(hr_dev, &desc, 1);
1486 }
1487
1488 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1489 {
1490         struct hns_roce_caps *caps = &hr_dev->caps;
1491         int ret;
1492
1493         ret = hns_roce_cmq_query_hw_info(hr_dev);
1494         if (ret) {
1495                 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
1496                         ret);
1497                 return ret;
1498         }
1499
1500         ret = hns_roce_query_fw_ver(hr_dev);
1501         if (ret) {
1502                 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1503                         ret);
1504                 return ret;
1505         }
1506
1507         ret = hns_roce_config_global_param(hr_dev);
1508         if (ret) {
1509                 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1510                         ret);
1511                 return ret;
1512         }
1513
1514         /* Get pf resource owned by every pf */
1515         ret = hns_roce_query_pf_resource(hr_dev);
1516         if (ret) {
1517                 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1518                         ret);
1519                 return ret;
1520         }
1521
1522         if (hr_dev->pci_dev->revision == 0x21) {
1523                 ret = hns_roce_query_pf_timer_resource(hr_dev);
1524                 if (ret) {
1525                         dev_err(hr_dev->dev,
1526                                 "Query pf timer resource fail, ret = %d.\n",
1527                                 ret);
1528                         return ret;
1529                 }
1530         }
1531
1532         ret = hns_roce_alloc_vf_resource(hr_dev);
1533         if (ret) {
1534                 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1535                         ret);
1536                 return ret;
1537         }
1538
1539         if (hr_dev->pci_dev->revision == 0x21) {
1540                 ret = hns_roce_set_vf_switch_param(hr_dev, 0);
1541                 if (ret) {
1542                         dev_err(hr_dev->dev,
1543                                 "Set function switch param fail, ret = %d.\n",
1544                                 ret);
1545                         return ret;
1546                 }
1547         }
1548
1549         hr_dev->vendor_part_id = hr_dev->pci_dev->device;
1550         hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
1551
1552         caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
1553         caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
1554         caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
1555         caps->num_srqs          = HNS_ROCE_V2_MAX_SRQ_NUM;
1556         caps->min_cqes          = HNS_ROCE_MIN_CQE_NUM;
1557         caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
1558         caps->max_srqwqes       = HNS_ROCE_V2_MAX_SRQWQE_NUM;
1559         caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1560         caps->max_extend_sg     = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1561         caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1562         caps->max_sq_inline     = HNS_ROCE_V2_MAX_SQ_INLINE;
1563         caps->max_srq_sg        = HNS_ROCE_V2_MAX_SRQ_SGE_NUM;
1564         caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
1565         caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
1566         caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
1567         caps->num_comp_vectors  = HNS_ROCE_V2_COMP_VEC_NUM;
1568         caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1569         caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
1570         caps->num_mtt_segs      = HNS_ROCE_V2_MAX_MTT_SEGS;
1571         caps->num_cqe_segs      = HNS_ROCE_V2_MAX_CQE_SEGS;
1572         caps->num_srqwqe_segs   = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
1573         caps->num_idx_segs      = HNS_ROCE_V2_MAX_IDX_SEGS;
1574         caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
1575         caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1576         caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1577         caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1578         caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1579         caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1580         caps->qpc_entry_sz      = HNS_ROCE_V2_QPC_ENTRY_SZ;
1581         caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1582         caps->trrl_entry_sz     = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1583         caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
1584         caps->srqc_entry_sz     = HNS_ROCE_V2_SRQC_ENTRY_SZ;
1585         caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1586         caps->mtt_entry_sz      = HNS_ROCE_V2_MTT_ENTRY_SZ;
1587         caps->idx_entry_sz      = 4;
1588         caps->cq_entry_sz       = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1589         caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1590         caps->reserved_lkey     = 0;
1591         caps->reserved_pds      = 0;
1592         caps->reserved_mrws     = 1;
1593         caps->reserved_uars     = 0;
1594         caps->reserved_cqs      = 0;
1595         caps->reserved_srqs     = 0;
1596         caps->reserved_qps      = HNS_ROCE_V2_RSV_QPS;
1597
1598         caps->qpc_ba_pg_sz      = 0;
1599         caps->qpc_buf_pg_sz     = 0;
1600         caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1601         caps->srqc_ba_pg_sz     = 0;
1602         caps->srqc_buf_pg_sz    = 0;
1603         caps->srqc_hop_num      = HNS_ROCE_CONTEXT_HOP_NUM;
1604         caps->cqc_ba_pg_sz      = 0;
1605         caps->cqc_buf_pg_sz     = 0;
1606         caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1607         caps->mpt_ba_pg_sz      = 0;
1608         caps->mpt_buf_pg_sz     = 0;
1609         caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1610         caps->pbl_ba_pg_sz      = 2;
1611         caps->pbl_buf_pg_sz     = 0;
1612         caps->pbl_hop_num       = HNS_ROCE_PBL_HOP_NUM;
1613         caps->mtt_ba_pg_sz      = 0;
1614         caps->mtt_buf_pg_sz     = 0;
1615         caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
1616         caps->wqe_sq_hop_num    = 2;
1617         caps->wqe_sge_hop_num   = 1;
1618         caps->wqe_rq_hop_num    = 2;
1619         caps->cqe_ba_pg_sz      = 6;
1620         caps->cqe_buf_pg_sz     = 0;
1621         caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
1622         caps->srqwqe_ba_pg_sz   = 0;
1623         caps->srqwqe_buf_pg_sz  = 0;
1624         caps->srqwqe_hop_num    = HNS_ROCE_SRQWQE_HOP_NUM;
1625         caps->idx_ba_pg_sz      = 0;
1626         caps->idx_buf_pg_sz     = 0;
1627         caps->idx_hop_num       = HNS_ROCE_IDX_HOP_NUM;
1628         caps->eqe_ba_pg_sz      = 0;
1629         caps->eqe_buf_pg_sz     = 0;
1630         caps->eqe_hop_num       = HNS_ROCE_EQE_HOP_NUM;
1631         caps->tsq_buf_pg_sz     = 0;
1632         caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1633
1634         caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
1635                                   HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1636                                   HNS_ROCE_CAP_FLAG_RQ_INLINE |
1637                                   HNS_ROCE_CAP_FLAG_RECORD_DB |
1638                                   HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1639
1640         if (hr_dev->pci_dev->revision == 0x21)
1641                 caps->flags |= HNS_ROCE_CAP_FLAG_MW |
1642                                HNS_ROCE_CAP_FLAG_FRMR;
1643
1644         caps->pkey_table_len[0] = 1;
1645         caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1646         caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
1647         caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
1648         caps->local_ca_ack_delay = 0;
1649         caps->max_mtu = IB_MTU_4096;
1650
1651         caps->max_srqs          = HNS_ROCE_V2_MAX_SRQ;
1652         caps->max_srq_wrs       = HNS_ROCE_V2_MAX_SRQ_WR;
1653         caps->max_srq_sges      = HNS_ROCE_V2_MAX_SRQ_SGE;
1654
1655         if (hr_dev->pci_dev->revision == 0x21) {
1656                 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC |
1657                                HNS_ROCE_CAP_FLAG_SRQ |
1658                                HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
1659
1660                 caps->num_qpc_timer       = HNS_ROCE_V2_MAX_QPC_TIMER_NUM;
1661                 caps->qpc_timer_entry_sz  = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
1662                 caps->qpc_timer_ba_pg_sz  = 0;
1663                 caps->qpc_timer_buf_pg_sz = 0;
1664                 caps->qpc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
1665                 caps->num_cqc_timer       = HNS_ROCE_V2_MAX_CQC_TIMER_NUM;
1666                 caps->cqc_timer_entry_sz  = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
1667                 caps->cqc_timer_ba_pg_sz  = 0;
1668                 caps->cqc_timer_buf_pg_sz = 0;
1669                 caps->cqc_timer_hop_num   = HNS_ROCE_HOP_NUM_0;
1670
1671                 caps->sccc_entry_sz     = HNS_ROCE_V2_SCCC_ENTRY_SZ;
1672                 caps->sccc_ba_pg_sz     = 0;
1673                 caps->sccc_buf_pg_sz    = 0;
1674                 caps->sccc_hop_num      = HNS_ROCE_SCCC_HOP_NUM;
1675         }
1676
1677         ret = hns_roce_v2_set_bt(hr_dev);
1678         if (ret)
1679                 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1680                         ret);
1681
1682         return ret;
1683 }
1684
1685 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
1686                                       enum hns_roce_link_table_type type)
1687 {
1688         struct hns_roce_cmq_desc desc[2];
1689         struct hns_roce_cfg_llm_a *req_a =
1690                                 (struct hns_roce_cfg_llm_a *)desc[0].data;
1691         struct hns_roce_cfg_llm_b *req_b =
1692                                 (struct hns_roce_cfg_llm_b *)desc[1].data;
1693         struct hns_roce_v2_priv *priv = hr_dev->priv;
1694         struct hns_roce_link_table *link_tbl;
1695         struct hns_roce_link_table_entry *entry;
1696         enum hns_roce_opcode_type opcode;
1697         u32 page_num;
1698         int i;
1699
1700         switch (type) {
1701         case TSQ_LINK_TABLE:
1702                 link_tbl = &priv->tsq;
1703                 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
1704                 break;
1705         case TPQ_LINK_TABLE:
1706                 link_tbl = &priv->tpq;
1707                 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
1708                 break;
1709         default:
1710                 return -EINVAL;
1711         }
1712
1713         page_num = link_tbl->npages;
1714         entry = link_tbl->table.buf;
1715         memset(req_a, 0, sizeof(*req_a));
1716         memset(req_b, 0, sizeof(*req_b));
1717
1718         for (i = 0; i < 2; i++) {
1719                 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
1720
1721                 if (i == 0)
1722                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1723                 else
1724                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1725
1726                 if (i == 0) {
1727                         req_a->base_addr_l = link_tbl->table.map & 0xffffffff;
1728                         req_a->base_addr_h = (link_tbl->table.map >> 32) &
1729                                              0xffffffff;
1730                         roce_set_field(req_a->depth_pgsz_init_en,
1731                                        CFG_LLM_QUE_DEPTH_M,
1732                                        CFG_LLM_QUE_DEPTH_S,
1733                                        link_tbl->npages);
1734                         roce_set_field(req_a->depth_pgsz_init_en,
1735                                        CFG_LLM_QUE_PGSZ_M,
1736                                        CFG_LLM_QUE_PGSZ_S,
1737                                        link_tbl->pg_sz);
1738                         req_a->head_ba_l = entry[0].blk_ba0;
1739                         req_a->head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr;
1740                         roce_set_field(req_a->head_ptr,
1741                                        CFG_LLM_HEAD_PTR_M,
1742                                        CFG_LLM_HEAD_PTR_S, 0);
1743                 } else {
1744                         req_b->tail_ba_l = entry[page_num - 1].blk_ba0;
1745                         roce_set_field(req_b->tail_ba_h,
1746                                        CFG_LLM_TAIL_BA_H_M,
1747                                        CFG_LLM_TAIL_BA_H_S,
1748                                        entry[page_num - 1].blk_ba1_nxt_ptr &
1749                                        HNS_ROCE_LINK_TABLE_BA1_M);
1750                         roce_set_field(req_b->tail_ptr,
1751                                        CFG_LLM_TAIL_PTR_M,
1752                                        CFG_LLM_TAIL_PTR_S,
1753                                        (entry[page_num - 2].blk_ba1_nxt_ptr &
1754                                        HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1755                                        HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1756                 }
1757         }
1758         roce_set_field(req_a->depth_pgsz_init_en,
1759                        CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
1760
1761         return hns_roce_cmq_send(hr_dev, desc, 2);
1762 }
1763
1764 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
1765                                     enum hns_roce_link_table_type type)
1766 {
1767         struct hns_roce_v2_priv *priv = hr_dev->priv;
1768         struct hns_roce_link_table *link_tbl;
1769         struct hns_roce_link_table_entry *entry;
1770         struct device *dev = hr_dev->dev;
1771         u32 buf_chk_sz;
1772         dma_addr_t t;
1773         int func_num = 1;
1774         int pg_num_a;
1775         int pg_num_b;
1776         int pg_num;
1777         int size;
1778         int i;
1779
1780         switch (type) {
1781         case TSQ_LINK_TABLE:
1782                 link_tbl = &priv->tsq;
1783                 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
1784                 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
1785                 pg_num_b = hr_dev->caps.sl_num * 4 + 2;
1786                 break;
1787         case TPQ_LINK_TABLE:
1788                 link_tbl = &priv->tpq;
1789                 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
1790                 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
1791                 pg_num_b = 2 * 4 * func_num + 2;
1792                 break;
1793         default:
1794                 return -EINVAL;
1795         }
1796
1797         pg_num = max(pg_num_a, pg_num_b);
1798         size = pg_num * sizeof(struct hns_roce_link_table_entry);
1799
1800         link_tbl->table.buf = dma_alloc_coherent(dev, size,
1801                                                  &link_tbl->table.map,
1802                                                  GFP_KERNEL);
1803         if (!link_tbl->table.buf)
1804                 goto out;
1805
1806         link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
1807                                     GFP_KERNEL);
1808         if (!link_tbl->pg_list)
1809                 goto err_kcalloc_failed;
1810
1811         entry = link_tbl->table.buf;
1812         for (i = 0; i < pg_num; ++i) {
1813                 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
1814                                                               &t, GFP_KERNEL);
1815                 if (!link_tbl->pg_list[i].buf)
1816                         goto err_alloc_buf_failed;
1817
1818                 link_tbl->pg_list[i].map = t;
1819
1820                 entry[i].blk_ba0 = (t >> 12) & 0xffffffff;
1821                 roce_set_field(entry[i].blk_ba1_nxt_ptr,
1822                                HNS_ROCE_LINK_TABLE_BA1_M,
1823                                HNS_ROCE_LINK_TABLE_BA1_S,
1824                                t >> 44);
1825
1826                 if (i < (pg_num - 1))
1827                         roce_set_field(entry[i].blk_ba1_nxt_ptr,
1828                                        HNS_ROCE_LINK_TABLE_NXT_PTR_M,
1829                                        HNS_ROCE_LINK_TABLE_NXT_PTR_S,
1830                                        i + 1);
1831         }
1832         link_tbl->npages = pg_num;
1833         link_tbl->pg_sz = buf_chk_sz;
1834
1835         return hns_roce_config_link_table(hr_dev, type);
1836
1837 err_alloc_buf_failed:
1838         for (i -= 1; i >= 0; i--)
1839                 dma_free_coherent(dev, buf_chk_sz,
1840                                   link_tbl->pg_list[i].buf,
1841                                   link_tbl->pg_list[i].map);
1842         kfree(link_tbl->pg_list);
1843
1844 err_kcalloc_failed:
1845         dma_free_coherent(dev, size, link_tbl->table.buf,
1846                           link_tbl->table.map);
1847
1848 out:
1849         return -ENOMEM;
1850 }
1851
1852 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
1853                                      struct hns_roce_link_table *link_tbl)
1854 {
1855         struct device *dev = hr_dev->dev;
1856         int size;
1857         int i;
1858
1859         size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
1860
1861         for (i = 0; i < link_tbl->npages; ++i)
1862                 if (link_tbl->pg_list[i].buf)
1863                         dma_free_coherent(dev, link_tbl->pg_sz,
1864                                           link_tbl->pg_list[i].buf,
1865                                           link_tbl->pg_list[i].map);
1866         kfree(link_tbl->pg_list);
1867
1868         dma_free_coherent(dev, size, link_tbl->table.buf,
1869                           link_tbl->table.map);
1870 }
1871
1872 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1873 {
1874         struct hns_roce_v2_priv *priv = hr_dev->priv;
1875         int qpc_count, cqc_count;
1876         int ret, i;
1877
1878         /* TSQ includes SQ doorbell and ack doorbell */
1879         ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
1880         if (ret) {
1881                 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
1882                 return ret;
1883         }
1884
1885         ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
1886         if (ret) {
1887                 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
1888                 goto err_tpq_init_failed;
1889         }
1890
1891         /* Alloc memory for QPC Timer buffer space chunk*/
1892         for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
1893              qpc_count++) {
1894                 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
1895                                          qpc_count);
1896                 if (ret) {
1897                         dev_err(hr_dev->dev, "QPC Timer get failed\n");
1898                         goto err_qpc_timer_failed;
1899                 }
1900         }
1901
1902         /* Alloc memory for CQC Timer buffer space chunk*/
1903         for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
1904              cqc_count++) {
1905                 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
1906                                          cqc_count);
1907                 if (ret) {
1908                         dev_err(hr_dev->dev, "CQC Timer get failed\n");
1909                         goto err_cqc_timer_failed;
1910                 }
1911         }
1912
1913         return 0;
1914
1915 err_cqc_timer_failed:
1916         for (i = 0; i < cqc_count; i++)
1917                 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
1918
1919 err_qpc_timer_failed:
1920         for (i = 0; i < qpc_count; i++)
1921                 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
1922
1923         hns_roce_free_link_table(hr_dev, &priv->tpq);
1924
1925 err_tpq_init_failed:
1926         hns_roce_free_link_table(hr_dev, &priv->tsq);
1927
1928         return ret;
1929 }
1930
1931 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
1932 {
1933         struct hns_roce_v2_priv *priv = hr_dev->priv;
1934
1935         if (hr_dev->pci_dev->revision == 0x21)
1936                 hns_roce_function_clear(hr_dev);
1937
1938         hns_roce_free_link_table(hr_dev, &priv->tpq);
1939         hns_roce_free_link_table(hr_dev, &priv->tsq);
1940 }
1941
1942 static int hns_roce_query_mbox_status(struct hns_roce_dev *hr_dev)
1943 {
1944         struct hns_roce_cmq_desc desc;
1945         struct hns_roce_mbox_status *mb_st =
1946                                        (struct hns_roce_mbox_status *)desc.data;
1947         enum hns_roce_cmd_return_status status;
1948
1949         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, true);
1950
1951         status = hns_roce_cmq_send(hr_dev, &desc, 1);
1952         if (status)
1953                 return status;
1954
1955         return cpu_to_le32(mb_st->mb_status_hw_run);
1956 }
1957
1958 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1959 {
1960         u32 status = hns_roce_query_mbox_status(hr_dev);
1961
1962         return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1963 }
1964
1965 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1966 {
1967         u32 status = hns_roce_query_mbox_status(hr_dev);
1968
1969         return status & HNS_ROCE_HW_MB_STATUS_MASK;
1970 }
1971
1972 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, u64 in_param,
1973                               u64 out_param, u32 in_modifier, u8 op_modifier,
1974                               u16 op, u16 token, int event)
1975 {
1976         struct hns_roce_cmq_desc desc;
1977         struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
1978
1979         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
1980
1981         mb->in_param_l = cpu_to_le64(in_param);
1982         mb->in_param_h = cpu_to_le64(in_param) >> 32;
1983         mb->out_param_l = cpu_to_le64(out_param);
1984         mb->out_param_h = cpu_to_le64(out_param) >> 32;
1985         mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op);
1986         mb->token_event_en = cpu_to_le32(event << 16 | token);
1987
1988         return hns_roce_cmq_send(hr_dev, &desc, 1);
1989 }
1990
1991 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1992                                  u64 out_param, u32 in_modifier, u8 op_modifier,
1993                                  u16 op, u16 token, int event)
1994 {
1995         struct device *dev = hr_dev->dev;
1996         unsigned long end;
1997         int ret;
1998
1999         end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
2000         while (hns_roce_v2_cmd_pending(hr_dev)) {
2001                 if (time_after(jiffies, end)) {
2002                         dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
2003                                 (int)end);
2004                         return -EAGAIN;
2005                 }
2006                 cond_resched();
2007         }
2008
2009         ret = hns_roce_mbox_post(hr_dev, in_param, out_param, in_modifier,
2010                                  op_modifier, op, token, event);
2011         if (ret)
2012                 dev_err(dev, "Post mailbox fail(%d)\n", ret);
2013
2014         return ret;
2015 }
2016
2017 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
2018                                 unsigned long timeout)
2019 {
2020         struct device *dev = hr_dev->dev;
2021         unsigned long end;
2022         u32 status;
2023
2024         end = msecs_to_jiffies(timeout) + jiffies;
2025         while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
2026                 cond_resched();
2027
2028         if (hns_roce_v2_cmd_pending(hr_dev)) {
2029                 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
2030                 return -ETIMEDOUT;
2031         }
2032
2033         status = hns_roce_v2_cmd_complete(hr_dev);
2034         if (status != 0x1) {
2035                 if (status == CMD_RST_PRC_EBUSY)
2036                         return status;
2037
2038                 dev_err(dev, "mailbox status 0x%x!\n", status);
2039                 return -EBUSY;
2040         }
2041
2042         return 0;
2043 }
2044
2045 static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
2046                                       int gid_index, const union ib_gid *gid,
2047                                       enum hns_roce_sgid_type sgid_type)
2048 {
2049         struct hns_roce_cmq_desc desc;
2050         struct hns_roce_cfg_sgid_tb *sgid_tb =
2051                                     (struct hns_roce_cfg_sgid_tb *)desc.data;
2052         u32 *p;
2053
2054         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
2055
2056         roce_set_field(sgid_tb->table_idx_rsv,
2057                        CFG_SGID_TB_TABLE_IDX_M,
2058                        CFG_SGID_TB_TABLE_IDX_S, gid_index);
2059         roce_set_field(sgid_tb->vf_sgid_type_rsv,
2060                        CFG_SGID_TB_VF_SGID_TYPE_M,
2061                        CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
2062
2063         p = (u32 *)&gid->raw[0];
2064         sgid_tb->vf_sgid_l = cpu_to_le32(*p);
2065
2066         p = (u32 *)&gid->raw[4];
2067         sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
2068
2069         p = (u32 *)&gid->raw[8];
2070         sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
2071
2072         p = (u32 *)&gid->raw[0xc];
2073         sgid_tb->vf_sgid_h = cpu_to_le32(*p);
2074
2075         return hns_roce_cmq_send(hr_dev, &desc, 1);
2076 }
2077
2078 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
2079                                int gid_index, const union ib_gid *gid,
2080                                const struct ib_gid_attr *attr)
2081 {
2082         enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
2083         int ret;
2084
2085         if (!gid || !attr)
2086                 return -EINVAL;
2087
2088         if (attr->gid_type == IB_GID_TYPE_ROCE)
2089                 sgid_type = GID_TYPE_FLAG_ROCE_V1;
2090
2091         if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
2092                 if (ipv6_addr_v4mapped((void *)gid))
2093                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
2094                 else
2095                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
2096         }
2097
2098         ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
2099         if (ret)
2100                 dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
2101
2102         return ret;
2103 }
2104
2105 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
2106                                u8 *addr)
2107 {
2108         struct hns_roce_cmq_desc desc;
2109         struct hns_roce_cfg_smac_tb *smac_tb =
2110                                     (struct hns_roce_cfg_smac_tb *)desc.data;
2111         u16 reg_smac_h;
2112         u32 reg_smac_l;
2113
2114         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
2115
2116         reg_smac_l = *(u32 *)(&addr[0]);
2117         reg_smac_h = *(u16 *)(&addr[4]);
2118
2119         memset(smac_tb, 0, sizeof(*smac_tb));
2120         roce_set_field(smac_tb->tb_idx_rsv,
2121                        CFG_SMAC_TB_IDX_M,
2122                        CFG_SMAC_TB_IDX_S, phy_port);
2123         roce_set_field(smac_tb->vf_smac_h_rsv,
2124                        CFG_SMAC_TB_VF_SMAC_H_M,
2125                        CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
2126         smac_tb->vf_smac_l = reg_smac_l;
2127
2128         return hns_roce_cmq_send(hr_dev, &desc, 1);
2129 }
2130
2131 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
2132                         struct hns_roce_mr *mr)
2133 {
2134         struct sg_dma_page_iter sg_iter;
2135         u64 page_addr;
2136         u64 *pages;
2137         int i;
2138
2139         mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2140         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2141         roce_set_field(mpt_entry->byte_48_mode_ba,
2142                        V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
2143                        upper_32_bits(mr->pbl_ba >> 3));
2144
2145         pages = (u64 *)__get_free_page(GFP_KERNEL);
2146         if (!pages)
2147                 return -ENOMEM;
2148
2149         i = 0;
2150         for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
2151                 page_addr = sg_page_iter_dma_address(&sg_iter);
2152                 pages[i] = page_addr >> 6;
2153
2154                 /* Record the first 2 entry directly to MTPT table */
2155                 if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
2156                         goto found;
2157                 i++;
2158         }
2159 found:
2160         mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
2161         roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
2162                        V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
2163
2164         mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
2165         roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
2166                        V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
2167         roce_set_field(mpt_entry->byte_64_buf_pa1,
2168                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2169                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2170                        mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2171
2172         free_page((unsigned long)pages);
2173
2174         return 0;
2175 }
2176
2177 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
2178                                   unsigned long mtpt_idx)
2179 {
2180         struct hns_roce_v2_mpt_entry *mpt_entry;
2181         int ret;
2182
2183         mpt_entry = mb_buf;
2184         memset(mpt_entry, 0, sizeof(*mpt_entry));
2185
2186         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2187                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2188         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2189                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
2190                        HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
2191         roce_set_field(mpt_entry->byte_4_pd_hop_st,
2192                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2193                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2194                        mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2195         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2196                        V2_MPT_BYTE_4_PD_S, mr->pd);
2197
2198         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
2199         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
2200         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2201         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
2202                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
2203         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
2204                      mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2205         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2206                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
2207         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2208                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
2209         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2210                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
2211
2212         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
2213                      mr->type == MR_TYPE_MR ? 0 : 1);
2214         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
2215                      1);
2216
2217         mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
2218         mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
2219         mpt_entry->lkey = cpu_to_le32(mr->key);
2220         mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
2221         mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
2222
2223         if (mr->type == MR_TYPE_DMA)
2224                 return 0;
2225
2226         ret = set_mtpt_pbl(mpt_entry, mr);
2227
2228         return ret;
2229 }
2230
2231 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
2232                                         struct hns_roce_mr *mr, int flags,
2233                                         u32 pdn, int mr_access_flags, u64 iova,
2234                                         u64 size, void *mb_buf)
2235 {
2236         struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
2237         int ret = 0;
2238
2239         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2240                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
2241
2242         if (flags & IB_MR_REREG_PD) {
2243                 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2244                                V2_MPT_BYTE_4_PD_S, pdn);
2245                 mr->pd = pdn;
2246         }
2247
2248         if (flags & IB_MR_REREG_ACCESS) {
2249                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2250                              V2_MPT_BYTE_8_BIND_EN_S,
2251                              (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
2252                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
2253                              V2_MPT_BYTE_8_ATOMIC_EN_S,
2254                              mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
2255                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
2256                              mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
2257                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
2258                              mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
2259                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
2260                              mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
2261         }
2262
2263         if (flags & IB_MR_REREG_TRANS) {
2264                 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
2265                 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
2266                 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
2267                 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
2268
2269                 mr->iova = iova;
2270                 mr->size = size;
2271
2272                 ret = set_mtpt_pbl(mpt_entry, mr);
2273         }
2274
2275         return ret;
2276 }
2277
2278 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
2279 {
2280         struct hns_roce_v2_mpt_entry *mpt_entry;
2281
2282         mpt_entry = mb_buf;
2283         memset(mpt_entry, 0, sizeof(*mpt_entry));
2284
2285         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2286                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2287         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2288                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
2289         roce_set_field(mpt_entry->byte_4_pd_hop_st,
2290                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2291                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2292                        mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2293         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2294                        V2_MPT_BYTE_4_PD_S, mr->pd);
2295
2296         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
2297         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2298         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2299
2300         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
2301         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2302         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
2303         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2304
2305         mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
2306
2307         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
2308         roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
2309                        V2_MPT_BYTE_48_PBL_BA_H_S,
2310                        upper_32_bits(mr->pbl_ba >> 3));
2311
2312         roce_set_field(mpt_entry->byte_64_buf_pa1,
2313                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2314                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2315                        mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2316
2317         return 0;
2318 }
2319
2320 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
2321 {
2322         struct hns_roce_v2_mpt_entry *mpt_entry;
2323
2324         mpt_entry = mb_buf;
2325         memset(mpt_entry, 0, sizeof(*mpt_entry));
2326
2327         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
2328                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
2329         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
2330                        V2_MPT_BYTE_4_PD_S, mw->pdn);
2331         roce_set_field(mpt_entry->byte_4_pd_hop_st,
2332                        V2_MPT_BYTE_4_PBL_HOP_NUM_M,
2333                        V2_MPT_BYTE_4_PBL_HOP_NUM_S,
2334                        mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ?
2335                        0 : mw->pbl_hop_num);
2336         roce_set_field(mpt_entry->byte_4_pd_hop_st,
2337                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
2338                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
2339                        mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
2340
2341         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
2342         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
2343
2344         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
2345         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
2346         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
2347         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
2348                      mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
2349
2350         roce_set_field(mpt_entry->byte_64_buf_pa1,
2351                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
2352                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
2353                        mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
2354
2355         mpt_entry->lkey = cpu_to_le32(mw->rkey);
2356
2357         return 0;
2358 }
2359
2360 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2361 {
2362         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
2363                                    n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
2364 }
2365
2366 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
2367 {
2368         struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
2369
2370         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
2371         return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
2372                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
2373 }
2374
2375 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
2376 {
2377         return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2378 }
2379
2380 static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
2381 {
2382         return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
2383 }
2384
2385 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
2386 {
2387         /* always called with interrupts disabled. */
2388         spin_lock(&srq->lock);
2389
2390         bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
2391         srq->tail++;
2392
2393         spin_unlock(&srq->lock);
2394 }
2395
2396 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2397 {
2398         *hr_cq->set_ci_db = cons_index & 0xffffff;
2399 }
2400
2401 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2402                                    struct hns_roce_srq *srq)
2403 {
2404         struct hns_roce_v2_cqe *cqe, *dest;
2405         u32 prod_index;
2406         int nfreed = 0;
2407         int wqe_index;
2408         u8 owner_bit;
2409
2410         for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
2411              ++prod_index) {
2412                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
2413                         break;
2414         }
2415
2416         /*
2417          * Now backwards through the CQ, removing CQ entries
2418          * that match our QP by overwriting them with next entries.
2419          */
2420         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2421                 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2422                 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2423                                     V2_CQE_BYTE_16_LCL_QPN_S) &
2424                                     HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
2425                         if (srq &&
2426                             roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S)) {
2427                                 wqe_index = roce_get_field(cqe->byte_4,
2428                                                      V2_CQE_BYTE_4_WQE_INDX_M,
2429                                                      V2_CQE_BYTE_4_WQE_INDX_S);
2430                                 hns_roce_free_srq_wqe(srq, wqe_index);
2431                         }
2432                         ++nfreed;
2433                 } else if (nfreed) {
2434                         dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
2435                                           hr_cq->ib_cq.cqe);
2436                         owner_bit = roce_get_bit(dest->byte_4,
2437                                                  V2_CQE_BYTE_4_OWNER_S);
2438                         memcpy(dest, cqe, sizeof(*cqe));
2439                         roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
2440                                      owner_bit);
2441                 }
2442         }
2443
2444         if (nfreed) {
2445                 hr_cq->cons_index += nfreed;
2446                 /*
2447                  * Make sure update of buffer contents is done before
2448                  * updating consumer index.
2449                  */
2450                 wmb();
2451                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2452         }
2453 }
2454
2455 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2456                                  struct hns_roce_srq *srq)
2457 {
2458         spin_lock_irq(&hr_cq->lock);
2459         __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
2460         spin_unlock_irq(&hr_cq->lock);
2461 }
2462
2463 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
2464                                   struct hns_roce_cq *hr_cq, void *mb_buf,
2465                                   u64 *mtts, dma_addr_t dma_handle, int nent,
2466                                   u32 vector)
2467 {
2468         struct hns_roce_v2_cq_context *cq_context;
2469
2470         cq_context = mb_buf;
2471         memset(cq_context, 0, sizeof(*cq_context));
2472
2473         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
2474                        V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
2475         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
2476                        V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
2477         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
2478                        V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
2479         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
2480                        V2_CQC_BYTE_4_CEQN_S, vector);
2481         cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
2482
2483         roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
2484                        V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
2485
2486         cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
2487         cq_context->cqe_cur_blk_addr =
2488                                 cpu_to_le32(cq_context->cqe_cur_blk_addr);
2489
2490         roce_set_field(cq_context->byte_16_hop_addr,
2491                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
2492                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
2493                        cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
2494         roce_set_field(cq_context->byte_16_hop_addr,
2495                        V2_CQC_BYTE_16_CQE_HOP_NUM_M,
2496                        V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
2497                        HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
2498
2499         cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
2500         roce_set_field(cq_context->byte_24_pgsz_addr,
2501                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
2502                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
2503                        cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
2504         roce_set_field(cq_context->byte_24_pgsz_addr,
2505                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
2506                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
2507                        hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
2508         roce_set_field(cq_context->byte_24_pgsz_addr,
2509                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
2510                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
2511                        hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
2512
2513         cq_context->cqe_ba = (u32)(dma_handle >> 3);
2514
2515         roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
2516                        V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
2517
2518         if (hr_cq->db_en)
2519                 roce_set_bit(cq_context->byte_44_db_record,
2520                              V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
2521
2522         roce_set_field(cq_context->byte_44_db_record,
2523                        V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
2524                        V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
2525                        ((u32)hr_cq->db.dma) >> 1);
2526         cq_context->db_record_addr = hr_cq->db.dma >> 32;
2527
2528         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2529                        V2_CQC_BYTE_56_CQ_MAX_CNT_M,
2530                        V2_CQC_BYTE_56_CQ_MAX_CNT_S,
2531                        HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
2532         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2533                        V2_CQC_BYTE_56_CQ_PERIOD_M,
2534                        V2_CQC_BYTE_56_CQ_PERIOD_S,
2535                        HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
2536 }
2537
2538 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
2539                                      enum ib_cq_notify_flags flags)
2540 {
2541         struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
2542         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2543         u32 notification_flag;
2544         u32 doorbell[2];
2545
2546         doorbell[0] = 0;
2547         doorbell[1] = 0;
2548
2549         notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
2550                              V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
2551         /*
2552          * flags = 0; Notification Flag = 1, next
2553          * flags = 1; Notification Flag = 0, solocited
2554          */
2555         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
2556                        hr_cq->cqn);
2557         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
2558                        HNS_ROCE_V2_CQ_DB_NTR);
2559         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
2560                        V2_CQ_DB_PARAMETER_CONS_IDX_S,
2561                        hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2562         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
2563                        V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
2564         roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
2565                      notification_flag);
2566
2567         hns_roce_write64(hr_dev, doorbell, hr_cq->cq_db_l);
2568
2569         return 0;
2570 }
2571
2572 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
2573                                                     struct hns_roce_qp **cur_qp,
2574                                                     struct ib_wc *wc)
2575 {
2576         struct hns_roce_rinl_sge *sge_list;
2577         u32 wr_num, wr_cnt, sge_num;
2578         u32 sge_cnt, data_len, size;
2579         void *wqe_buf;
2580
2581         wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
2582                                 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
2583         wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
2584
2585         sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2586         sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2587         wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
2588         data_len = wc->byte_len;
2589
2590         for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
2591                 size = min(sge_list[sge_cnt].len, data_len);
2592                 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
2593
2594                 data_len -= size;
2595                 wqe_buf += size;
2596         }
2597
2598         if (data_len) {
2599                 wc->status = IB_WC_LOC_LEN_ERR;
2600                 return -EAGAIN;
2601         }
2602
2603         return 0;
2604 }
2605
2606 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2607                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2608 {
2609         struct hns_roce_srq *srq = NULL;
2610         struct hns_roce_dev *hr_dev;
2611         struct hns_roce_v2_cqe *cqe;
2612         struct hns_roce_qp *hr_qp;
2613         struct hns_roce_wq *wq;
2614         struct ib_qp_attr attr;
2615         int attr_mask;
2616         int is_send;
2617         u16 wqe_ctr;
2618         u32 opcode;
2619         u32 status;
2620         int qpn;
2621         int ret;
2622
2623         /* Find cqe according to consumer index */
2624         cqe = next_cqe_sw_v2(hr_cq);
2625         if (!cqe)
2626                 return -EAGAIN;
2627
2628         ++hr_cq->cons_index;
2629         /* Memory barrier */
2630         rmb();
2631
2632         /* 0->SQ, 1->RQ */
2633         is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
2634
2635         qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2636                                 V2_CQE_BYTE_16_LCL_QPN_S);
2637
2638         if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2639                 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2640                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2641                 if (unlikely(!hr_qp)) {
2642                         dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2643                                 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
2644                         return -EINVAL;
2645                 }
2646                 *cur_qp = hr_qp;
2647         }
2648
2649         wc->qp = &(*cur_qp)->ibqp;
2650         wc->vendor_err = 0;
2651
2652         if (is_send) {
2653                 wq = &(*cur_qp)->sq;
2654                 if ((*cur_qp)->sq_signal_bits) {
2655                         /*
2656                          * If sg_signal_bit is 1,
2657                          * firstly tail pointer updated to wqe
2658                          * which current cqe correspond to
2659                          */
2660                         wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2661                                                       V2_CQE_BYTE_4_WQE_INDX_M,
2662                                                       V2_CQE_BYTE_4_WQE_INDX_S);
2663                         wq->tail += (wqe_ctr - (u16)wq->tail) &
2664                                     (wq->wqe_cnt - 1);
2665                 }
2666
2667                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2668                 ++wq->tail;
2669         } else if ((*cur_qp)->ibqp.srq) {
2670                 srq = to_hr_srq((*cur_qp)->ibqp.srq);
2671                 wqe_ctr = le16_to_cpu(roce_get_field(cqe->byte_4,
2672                                                      V2_CQE_BYTE_4_WQE_INDX_M,
2673                                                      V2_CQE_BYTE_4_WQE_INDX_S));
2674                 wc->wr_id = srq->wrid[wqe_ctr];
2675                 hns_roce_free_srq_wqe(srq, wqe_ctr);
2676         } else {
2677                 /* Update tail pointer, record wr_id */
2678                 wq = &(*cur_qp)->rq;
2679                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2680                 ++wq->tail;
2681         }
2682
2683         status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2684                                 V2_CQE_BYTE_4_STATUS_S);
2685         switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2686         case HNS_ROCE_CQE_V2_SUCCESS:
2687                 wc->status = IB_WC_SUCCESS;
2688                 break;
2689         case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2690                 wc->status = IB_WC_LOC_LEN_ERR;
2691                 break;
2692         case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2693                 wc->status = IB_WC_LOC_QP_OP_ERR;
2694                 break;
2695         case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2696                 wc->status = IB_WC_LOC_PROT_ERR;
2697                 break;
2698         case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2699                 wc->status = IB_WC_WR_FLUSH_ERR;
2700                 break;
2701         case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2702                 wc->status = IB_WC_MW_BIND_ERR;
2703                 break;
2704         case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2705                 wc->status = IB_WC_BAD_RESP_ERR;
2706                 break;
2707         case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2708                 wc->status = IB_WC_LOC_ACCESS_ERR;
2709                 break;
2710         case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2711                 wc->status = IB_WC_REM_INV_REQ_ERR;
2712                 break;
2713         case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2714                 wc->status = IB_WC_REM_ACCESS_ERR;
2715                 break;
2716         case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2717                 wc->status = IB_WC_REM_OP_ERR;
2718                 break;
2719         case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2720                 wc->status = IB_WC_RETRY_EXC_ERR;
2721                 break;
2722         case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2723                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2724                 break;
2725         case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2726                 wc->status = IB_WC_REM_ABORT_ERR;
2727                 break;
2728         default:
2729                 wc->status = IB_WC_GENERAL_ERR;
2730                 break;
2731         }
2732
2733         /* flush cqe if wc status is error, excluding flush error */
2734         if ((wc->status != IB_WC_SUCCESS) &&
2735             (wc->status != IB_WC_WR_FLUSH_ERR)) {
2736                 attr_mask = IB_QP_STATE;
2737                 attr.qp_state = IB_QPS_ERR;
2738                 return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2739                                              &attr, attr_mask,
2740                                              (*cur_qp)->state, IB_QPS_ERR);
2741         }
2742
2743         if (wc->status == IB_WC_WR_FLUSH_ERR)
2744                 return 0;
2745
2746         if (is_send) {
2747                 wc->wc_flags = 0;
2748                 /* SQ corresponding to CQE */
2749                 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2750                                        V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2751                 case HNS_ROCE_SQ_OPCODE_SEND:
2752                         wc->opcode = IB_WC_SEND;
2753                         break;
2754                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2755                         wc->opcode = IB_WC_SEND;
2756                         break;
2757                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2758                         wc->opcode = IB_WC_SEND;
2759                         wc->wc_flags |= IB_WC_WITH_IMM;
2760                         break;
2761                 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2762                         wc->opcode = IB_WC_RDMA_READ;
2763                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2764                         break;
2765                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2766                         wc->opcode = IB_WC_RDMA_WRITE;
2767                         break;
2768                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2769                         wc->opcode = IB_WC_RDMA_WRITE;
2770                         wc->wc_flags |= IB_WC_WITH_IMM;
2771                         break;
2772                 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2773                         wc->opcode = IB_WC_LOCAL_INV;
2774                         wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2775                         break;
2776                 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2777                         wc->opcode = IB_WC_COMP_SWAP;
2778                         wc->byte_len  = 8;
2779                         break;
2780                 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2781                         wc->opcode = IB_WC_FETCH_ADD;
2782                         wc->byte_len  = 8;
2783                         break;
2784                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2785                         wc->opcode = IB_WC_MASKED_COMP_SWAP;
2786                         wc->byte_len  = 8;
2787                         break;
2788                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2789                         wc->opcode = IB_WC_MASKED_FETCH_ADD;
2790                         wc->byte_len  = 8;
2791                         break;
2792                 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2793                         wc->opcode = IB_WC_REG_MR;
2794                         break;
2795                 case HNS_ROCE_SQ_OPCODE_BIND_MW:
2796                         wc->opcode = IB_WC_REG_MR;
2797                         break;
2798                 default:
2799                         wc->status = IB_WC_GENERAL_ERR;
2800                         break;
2801                 }
2802         } else {
2803                 /* RQ correspond to CQE */
2804                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2805
2806                 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2807                                         V2_CQE_BYTE_4_OPCODE_S);
2808                 switch (opcode & 0x1f) {
2809                 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
2810                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2811                         wc->wc_flags = IB_WC_WITH_IMM;
2812                         wc->ex.imm_data =
2813                                 cpu_to_be32(le32_to_cpu(cqe->immtdata));
2814                         break;
2815                 case HNS_ROCE_V2_OPCODE_SEND:
2816                         wc->opcode = IB_WC_RECV;
2817                         wc->wc_flags = 0;
2818                         break;
2819                 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
2820                         wc->opcode = IB_WC_RECV;
2821                         wc->wc_flags = IB_WC_WITH_IMM;
2822                         wc->ex.imm_data =
2823                                 cpu_to_be32(le32_to_cpu(cqe->immtdata));
2824                         break;
2825                 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
2826                         wc->opcode = IB_WC_RECV;
2827                         wc->wc_flags = IB_WC_WITH_INVALIDATE;
2828                         wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
2829                         break;
2830                 default:
2831                         wc->status = IB_WC_GENERAL_ERR;
2832                         break;
2833                 }
2834
2835                 if ((wc->qp->qp_type == IB_QPT_RC ||
2836                      wc->qp->qp_type == IB_QPT_UC) &&
2837                     (opcode == HNS_ROCE_V2_OPCODE_SEND ||
2838                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
2839                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2840                     (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2841                         ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2842                         if (ret)
2843                                 return -EAGAIN;
2844                 }
2845
2846                 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2847                                             V2_CQE_BYTE_32_SL_S);
2848                 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
2849                                                 V2_CQE_BYTE_32_RMT_QPN_M,
2850                                                 V2_CQE_BYTE_32_RMT_QPN_S);
2851                 wc->slid = 0;
2852                 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
2853                                               V2_CQE_BYTE_32_GRH_S) ?
2854                                               IB_WC_GRH : 0);
2855                 wc->port_num = roce_get_field(cqe->byte_32,
2856                                 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2857                 wc->pkey_index = 0;
2858                 memcpy(wc->smac, cqe->smac, 4);
2859                 wc->smac[4] = roce_get_field(cqe->byte_28,
2860                                              V2_CQE_BYTE_28_SMAC_4_M,
2861                                              V2_CQE_BYTE_28_SMAC_4_S);
2862                 wc->smac[5] = roce_get_field(cqe->byte_28,
2863                                              V2_CQE_BYTE_28_SMAC_5_M,
2864                                              V2_CQE_BYTE_28_SMAC_5_S);
2865                 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
2866                         wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
2867                                                           V2_CQE_BYTE_28_VID_M,
2868                                                           V2_CQE_BYTE_28_VID_S);
2869                 } else {
2870                         wc->vlan_id = 0xffff;
2871                 }
2872
2873                 wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
2874                 wc->network_hdr_type = roce_get_field(cqe->byte_28,
2875                                                     V2_CQE_BYTE_28_PORT_TYPE_M,
2876                                                     V2_CQE_BYTE_28_PORT_TYPE_S);
2877         }
2878
2879         return 0;
2880 }
2881
2882 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2883                                struct ib_wc *wc)
2884 {
2885         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2886         struct hns_roce_qp *cur_qp = NULL;
2887         unsigned long flags;
2888         int npolled;
2889
2890         spin_lock_irqsave(&hr_cq->lock, flags);
2891
2892         for (npolled = 0; npolled < num_entries; ++npolled) {
2893                 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
2894                         break;
2895         }
2896
2897         if (npolled) {
2898                 /* Memory barrier */
2899                 wmb();
2900                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2901         }
2902
2903         spin_unlock_irqrestore(&hr_cq->lock, flags);
2904
2905         return npolled;
2906 }
2907
2908 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
2909                                struct hns_roce_hem_table *table, int obj,
2910                                int step_idx)
2911 {
2912         struct device *dev = hr_dev->dev;
2913         struct hns_roce_cmd_mailbox *mailbox;
2914         struct hns_roce_hem_iter iter;
2915         struct hns_roce_hem_mhop mhop;
2916         struct hns_roce_hem *hem;
2917         unsigned long mhop_obj = obj;
2918         int i, j, k;
2919         int ret = 0;
2920         u64 hem_idx = 0;
2921         u64 l1_idx = 0;
2922         u64 bt_ba = 0;
2923         u32 chunk_ba_num;
2924         u32 hop_num;
2925         u16 op = 0xff;
2926
2927         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2928                 return 0;
2929
2930         hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
2931         i = mhop.l0_idx;
2932         j = mhop.l1_idx;
2933         k = mhop.l2_idx;
2934         hop_num = mhop.hop_num;
2935         chunk_ba_num = mhop.bt_chunk_size / 8;
2936
2937         if (hop_num == 2) {
2938                 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
2939                           k;
2940                 l1_idx = i * chunk_ba_num + j;
2941         } else if (hop_num == 1) {
2942                 hem_idx = i * chunk_ba_num + j;
2943         } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
2944                 hem_idx = i;
2945         }
2946
2947         switch (table->type) {
2948         case HEM_TYPE_QPC:
2949                 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2950                 break;
2951         case HEM_TYPE_MTPT:
2952                 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2953                 break;
2954         case HEM_TYPE_CQC:
2955                 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2956                 break;
2957         case HEM_TYPE_SRQC:
2958                 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2959                 break;
2960         case HEM_TYPE_SCCC:
2961                 op = HNS_ROCE_CMD_WRITE_SCCC_BT0;
2962                 break;
2963         case HEM_TYPE_QPC_TIMER:
2964                 op = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
2965                 break;
2966         case HEM_TYPE_CQC_TIMER:
2967                 op = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
2968                 break;
2969         default:
2970                 dev_warn(dev, "Table %d not to be written by mailbox!\n",
2971                          table->type);
2972                 return 0;
2973         }
2974
2975         if (table->type == HEM_TYPE_SCCC && step_idx)
2976                 return 0;
2977
2978         op += step_idx;
2979
2980         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2981         if (IS_ERR(mailbox))
2982                 return PTR_ERR(mailbox);
2983
2984         if (table->type == HEM_TYPE_SCCC)
2985                 obj = mhop.l0_idx;
2986
2987         if (check_whether_last_step(hop_num, step_idx)) {
2988                 hem = table->hem[hem_idx];
2989                 for (hns_roce_hem_first(hem, &iter);
2990                      !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2991                         bt_ba = hns_roce_hem_addr(&iter);
2992
2993                         /* configure the ba, tag, and op */
2994                         ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2995                                                 obj, 0, op,
2996                                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2997                 }
2998         } else {
2999                 if (step_idx == 0)
3000                         bt_ba = table->bt_l0_dma_addr[i];
3001                 else if (step_idx == 1 && hop_num == 2)
3002                         bt_ba = table->bt_l1_dma_addr[l1_idx];
3003
3004                 /* configure the ba, tag, and op */
3005                 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
3006                                         0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
3007         }
3008
3009         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3010         return ret;
3011 }
3012
3013 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
3014                                  struct hns_roce_hem_table *table, int obj,
3015                                  int step_idx)
3016 {
3017         struct device *dev = hr_dev->dev;
3018         struct hns_roce_cmd_mailbox *mailbox;
3019         int ret;
3020         u16 op = 0xff;
3021
3022         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
3023                 return 0;
3024
3025         switch (table->type) {
3026         case HEM_TYPE_QPC:
3027                 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
3028                 break;
3029         case HEM_TYPE_MTPT:
3030                 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
3031                 break;
3032         case HEM_TYPE_CQC:
3033                 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
3034                 break;
3035         case HEM_TYPE_SCCC:
3036         case HEM_TYPE_QPC_TIMER:
3037         case HEM_TYPE_CQC_TIMER:
3038                 break;
3039         case HEM_TYPE_SRQC:
3040                 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
3041                 break;
3042         default:
3043                 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
3044                          table->type);
3045                 return 0;
3046         }
3047
3048         if (table->type == HEM_TYPE_SCCC ||
3049             table->type == HEM_TYPE_QPC_TIMER ||
3050             table->type == HEM_TYPE_CQC_TIMER)
3051                 return 0;
3052
3053         op += step_idx;
3054
3055         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3056         if (IS_ERR(mailbox))
3057                 return PTR_ERR(mailbox);
3058
3059         /* configure the tag and op */
3060         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
3061                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
3062
3063         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3064         return ret;
3065 }
3066
3067 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
3068                                  enum ib_qp_state cur_state,
3069                                  enum ib_qp_state new_state,
3070                                  struct hns_roce_v2_qp_context *context,
3071                                  struct hns_roce_qp *hr_qp)
3072 {
3073         struct hns_roce_cmd_mailbox *mailbox;
3074         int ret;
3075
3076         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3077         if (IS_ERR(mailbox))
3078                 return PTR_ERR(mailbox);
3079
3080         memcpy(mailbox->buf, context, sizeof(*context) * 2);
3081
3082         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
3083                                 HNS_ROCE_CMD_MODIFY_QPC,
3084                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
3085
3086         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3087
3088         return ret;
3089 }
3090
3091 static void set_access_flags(struct hns_roce_qp *hr_qp,
3092                              struct hns_roce_v2_qp_context *context,
3093                              struct hns_roce_v2_qp_context *qpc_mask,
3094                              const struct ib_qp_attr *attr, int attr_mask)
3095 {
3096         u8 dest_rd_atomic;
3097         u32 access_flags;
3098
3099         dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
3100                          attr->max_dest_rd_atomic : hr_qp->resp_depth;
3101
3102         access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
3103                        attr->qp_access_flags : hr_qp->atomic_rd_en;
3104
3105         if (!dest_rd_atomic)
3106                 access_flags &= IB_ACCESS_REMOTE_WRITE;
3107
3108         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3109                      !!(access_flags & IB_ACCESS_REMOTE_READ));
3110         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
3111
3112         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3113                      !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3114         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
3115
3116         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3117                      !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
3118         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
3119 }
3120
3121 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
3122                                     const struct ib_qp_attr *attr,
3123                                     int attr_mask,
3124                                     struct hns_roce_v2_qp_context *context,
3125                                     struct hns_roce_v2_qp_context *qpc_mask)
3126 {
3127         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3128         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3129
3130         /*
3131          * In v2 engine, software pass context and context mask to hardware
3132          * when modifying qp. If software need modify some fields in context,
3133          * we should set all bits of the relevant fields in context mask to
3134          * 0 at the same time, else set them to 0x1.
3135          */
3136         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3137                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3138         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3139                        V2_QPC_BYTE_4_TST_S, 0);
3140
3141         if (ibqp->qp_type == IB_QPT_GSI)
3142                 roce_set_field(context->byte_4_sqpn_tst,
3143                                V2_QPC_BYTE_4_SGE_SHIFT_M,
3144                                V2_QPC_BYTE_4_SGE_SHIFT_S,
3145                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
3146         else
3147                 roce_set_field(context->byte_4_sqpn_tst,
3148                                V2_QPC_BYTE_4_SGE_SHIFT_M,
3149                                V2_QPC_BYTE_4_SGE_SHIFT_S,
3150                                hr_qp->sq.max_gs > 2 ?
3151                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3152
3153         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3154                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3155
3156         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3157                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3158         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3159                        V2_QPC_BYTE_4_SQPN_S, 0);
3160
3161         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3162                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3163         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3164                        V2_QPC_BYTE_16_PD_S, 0);
3165
3166         roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3167                        V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
3168         roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
3169                        V2_QPC_BYTE_20_RQWS_S, 0);
3170
3171         roce_set_field(context->byte_20_smac_sgid_idx,
3172                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3173                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3174         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3175                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3176
3177         roce_set_field(context->byte_20_smac_sgid_idx,
3178                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3179                        (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3180                        hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 :
3181                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3182         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3183                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3184
3185         /* No VLAN need to set 0xFFF */
3186         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3187                        V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
3188         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
3189                        V2_QPC_BYTE_24_VLAN_ID_S, 0);
3190
3191         /*
3192          * Set some fields in context to zero, Because the default values
3193          * of all fields in context are zero, we need not set them to 0 again.
3194          * but we should set the relevant fields of context mask to 0.
3195          */
3196         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
3197         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
3198         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
3199         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
3200
3201         roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,
3202                        V2_QPC_BYTE_60_TEMPID_S, 0);
3203
3204         roce_set_field(qpc_mask->byte_60_qpst_tempid,
3205                        V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,
3206                        0);
3207         roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3208                      V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);
3209         roce_set_bit(qpc_mask->byte_60_qpst_tempid,
3210                      V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);
3211         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
3212         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
3213
3214         if (hr_qp->rdb_en) {
3215                 roce_set_bit(context->byte_68_rq_db,
3216                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
3217                 roce_set_bit(qpc_mask->byte_68_rq_db,
3218                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
3219         }
3220
3221         roce_set_field(context->byte_68_rq_db,
3222                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3223                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
3224                        ((u32)hr_qp->rdb.dma) >> 1);
3225         roce_set_field(qpc_mask->byte_68_rq_db,
3226                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
3227                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
3228         context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
3229         qpc_mask->rq_db_record_addr = 0;
3230
3231         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
3232                     (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
3233         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
3234
3235         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3236                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3237         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3238                        V2_QPC_BYTE_80_RX_CQN_S, 0);
3239         if (ibqp->srq) {
3240                 roce_set_field(context->byte_76_srqn_op_en,
3241                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3242                                to_hr_srq(ibqp->srq)->srqn);
3243                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3244                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3245                 roce_set_bit(context->byte_76_srqn_op_en,
3246                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
3247                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3248                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
3249         }
3250
3251         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3252                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3253                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3254         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3255                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3256                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3257
3258         roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
3259                        V2_QPC_BYTE_92_SRQ_INFO_S, 0);
3260
3261         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3262                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3263
3264         roce_set_field(qpc_mask->byte_104_rq_sge,
3265                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
3266                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
3267
3268         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3269                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3270         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3271                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3272                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3273         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3274                      V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
3275
3276         qpc_mask->rq_rnr_timer = 0;
3277         qpc_mask->rx_msg_len = 0;
3278         qpc_mask->rx_rkey_pkt_info = 0;
3279         qpc_mask->rx_va = 0;
3280
3281         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3282                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3283         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3284                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3285
3286         roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,
3287                      0);
3288         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
3289                        V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
3290         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
3291                        V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
3292
3293         roce_set_field(qpc_mask->byte_144_raq,
3294                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
3295                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
3296         roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
3297                        V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
3298         roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
3299
3300         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
3301                        V2_QPC_BYTE_148_RQ_MSN_S, 0);
3302         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
3303                        V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
3304
3305         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3306                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3307         roce_set_field(qpc_mask->byte_152_raq,
3308                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
3309                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
3310
3311         roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
3312                        V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
3313
3314         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3315                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3316                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3317         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3318                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
3319                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
3320
3321         roce_set_bit(qpc_mask->byte_168_irrl_idx,
3322                      V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);
3323         roce_set_bit(qpc_mask->byte_168_irrl_idx,
3324                      V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);
3325         roce_set_bit(qpc_mask->byte_168_irrl_idx,
3326                      V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);
3327         roce_set_bit(qpc_mask->byte_168_irrl_idx,
3328                      V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
3329         roce_set_bit(qpc_mask->byte_168_irrl_idx,
3330                      V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
3331         roce_set_field(qpc_mask->byte_168_irrl_idx,
3332                        V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
3333                        V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
3334
3335         roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3336                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
3337         roce_set_field(qpc_mask->byte_172_sq_psn,
3338                        V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
3339                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
3340
3341         roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
3342                      0);
3343
3344         roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
3345         roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);
3346
3347         roce_set_field(qpc_mask->byte_176_msg_pktn,
3348                        V2_QPC_BYTE_176_MSG_USE_PKTN_M,
3349                        V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
3350         roce_set_field(qpc_mask->byte_176_msg_pktn,
3351                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
3352                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
3353
3354         roce_set_field(qpc_mask->byte_184_irrl_idx,
3355                        V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
3356                        V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
3357
3358         qpc_mask->cur_sge_offset = 0;
3359
3360         roce_set_field(qpc_mask->byte_192_ext_sge,
3361                        V2_QPC_BYTE_192_CUR_SGE_IDX_M,
3362                        V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
3363         roce_set_field(qpc_mask->byte_192_ext_sge,
3364                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
3365                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
3366
3367         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3368                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3369
3370         roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
3371                        V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
3372         roce_set_field(qpc_mask->byte_200_sq_max,
3373                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
3374                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
3375
3376         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
3377         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
3378
3379         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3380                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3381
3382         qpc_mask->sq_timer = 0;
3383
3384         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3385                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3386                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3387         roce_set_field(qpc_mask->byte_232_irrl_sge,
3388                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3389                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3390
3391         roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,
3392                      0);
3393         roce_set_bit(qpc_mask->byte_232_irrl_sge,
3394                      V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);
3395         roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,
3396                      0);
3397
3398         qpc_mask->irrl_cur_sge_offset = 0;
3399
3400         roce_set_field(qpc_mask->byte_240_irrl_tail,
3401                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3402                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3403         roce_set_field(qpc_mask->byte_240_irrl_tail,
3404                        V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
3405                        V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
3406         roce_set_field(qpc_mask->byte_240_irrl_tail,
3407                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
3408                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3409
3410         roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
3411                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3412         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
3413                      0);
3414         roce_set_field(qpc_mask->byte_248_ack_psn,
3415                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3416                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3417         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
3418                      0);
3419         roce_set_bit(qpc_mask->byte_248_ack_psn,
3420                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3421         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
3422                      0);
3423
3424         hr_qp->access_flags = attr->qp_access_flags;
3425         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3426                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3427         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3428                        V2_QPC_BYTE_252_TX_CQN_S, 0);
3429
3430         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
3431                        V2_QPC_BYTE_252_ERR_TYPE_S, 0);
3432
3433         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3434                        V2_QPC_BYTE_256_RQ_CQE_IDX_M,
3435                        V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
3436         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3437                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
3438                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
3439 }
3440
3441 static void modify_qp_init_to_init(struct ib_qp *ibqp,
3442                                    const struct ib_qp_attr *attr, int attr_mask,
3443                                    struct hns_roce_v2_qp_context *context,
3444                                    struct hns_roce_v2_qp_context *qpc_mask)
3445 {
3446         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3447
3448         /*
3449          * In v2 engine, software pass context and context mask to hardware
3450          * when modifying qp. If software need modify some fields in context,
3451          * we should set all bits of the relevant fields in context mask to
3452          * 0 at the same time, else set them to 0x1.
3453          */
3454         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3455                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3456         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3457                        V2_QPC_BYTE_4_TST_S, 0);
3458
3459         if (ibqp->qp_type == IB_QPT_GSI)
3460                 roce_set_field(context->byte_4_sqpn_tst,
3461                                V2_QPC_BYTE_4_SGE_SHIFT_M,
3462                                V2_QPC_BYTE_4_SGE_SHIFT_S,
3463                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
3464         else
3465                 roce_set_field(context->byte_4_sqpn_tst,
3466                                V2_QPC_BYTE_4_SGE_SHIFT_M,
3467                                V2_QPC_BYTE_4_SGE_SHIFT_S,
3468                                hr_qp->sq.max_gs >
3469                                HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
3470                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3471
3472         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3473                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3474
3475         if (attr_mask & IB_QP_ACCESS_FLAGS) {
3476                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3477                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3478                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3479                              0);
3480
3481                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3482                              !!(attr->qp_access_flags &
3483                              IB_ACCESS_REMOTE_WRITE));
3484                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3485                              0);
3486
3487                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3488                              !!(attr->qp_access_flags &
3489                              IB_ACCESS_REMOTE_ATOMIC));
3490                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3491                              0);
3492         } else {
3493                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3494                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3495                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3496                              0);
3497
3498                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3499                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3500                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3501                              0);
3502
3503                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3504                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3505                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3506                              0);
3507         }
3508
3509         roce_set_field(context->byte_20_smac_sgid_idx,
3510                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3511                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3512         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3513                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3514
3515         roce_set_field(context->byte_20_smac_sgid_idx,
3516                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3517                        (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
3518                        hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT || ibqp->srq) ? 0 :
3519                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3520         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3521                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3522
3523         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3524                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3525         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3526                        V2_QPC_BYTE_16_PD_S, 0);
3527
3528         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3529                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3530         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3531                        V2_QPC_BYTE_80_RX_CQN_S, 0);
3532
3533         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3534                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3535         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3536                        V2_QPC_BYTE_252_TX_CQN_S, 0);
3537
3538         if (ibqp->srq) {
3539                 roce_set_bit(context->byte_76_srqn_op_en,
3540                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
3541                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3542                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
3543                 roce_set_field(context->byte_76_srqn_op_en,
3544                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3545                                to_hr_srq(ibqp->srq)->srqn);
3546                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3547                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3548         }
3549
3550         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3551                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3552         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3553                        V2_QPC_BYTE_4_SQPN_S, 0);
3554
3555         if (attr_mask & IB_QP_DEST_QPN) {
3556                 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3557                                V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
3558                 roce_set_field(qpc_mask->byte_56_dqpn_err,
3559                                V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3560         }
3561 }
3562
3563 static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
3564                                    struct hns_roce_qp *hr_qp, int mtt_cnt,
3565                                    u32 page_size)
3566 {
3567         struct device *dev = hr_dev->dev;
3568
3569         if (hr_qp->rq.wqe_cnt < 1)
3570                 return true;
3571
3572         if (mtt_cnt < 1) {
3573                 dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n",
3574                         hr_qp->qpn);
3575                 return false;
3576         }
3577
3578         if (mtt_cnt < MTT_MIN_COUNT &&
3579                 (hr_qp->rq.offset + page_size) < hr_qp->buff_size) {
3580                 dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n",
3581                         hr_qp->qpn);
3582                 return false;
3583         }
3584
3585         return true;
3586 }
3587
3588 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
3589                                  const struct ib_qp_attr *attr, int attr_mask,
3590                                  struct hns_roce_v2_qp_context *context,
3591                                  struct hns_roce_v2_qp_context *qpc_mask)
3592 {
3593         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
3594         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3595         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3596         struct device *dev = hr_dev->dev;
3597         u64 mtts[MTT_MIN_COUNT] = { 0 };
3598         dma_addr_t dma_handle_3;
3599         dma_addr_t dma_handle_2;
3600         u64 wqe_sge_ba;
3601         u32 page_size;
3602         u8 port_num;
3603         u64 *mtts_3;
3604         u64 *mtts_2;
3605         int count;
3606         u8 *dmac;
3607         u8 *smac;
3608         int port;
3609
3610         /* Search qp buf's mtts */
3611         page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3612         count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3613                                   hr_qp->rq.offset / page_size, mtts,
3614                                   MTT_MIN_COUNT, &wqe_sge_ba);
3615         if (!ibqp->srq)
3616                 if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
3617                         return -EINVAL;
3618
3619         /* Search IRRL's mtts */
3620         mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
3621                                      hr_qp->qpn, &dma_handle_2);
3622         if (!mtts_2) {
3623                 dev_err(dev, "qp irrl_table find failed\n");
3624                 return -EINVAL;
3625         }
3626
3627         /* Search TRRL's mtts */
3628         mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3629                                      hr_qp->qpn, &dma_handle_3);
3630         if (!mtts_3) {
3631                 dev_err(dev, "qp trrl_table find failed\n");
3632                 return -EINVAL;
3633         }
3634
3635         if (attr_mask & IB_QP_ALT_PATH) {
3636                 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
3637                 return -EINVAL;
3638         }
3639
3640         dmac = (u8 *)attr->ah_attr.roce.dmac;
3641         context->wqe_sge_ba = (u32)(wqe_sge_ba >> 3);
3642         qpc_mask->wqe_sge_ba = 0;
3643
3644         /*
3645          * In v2 engine, software pass context and context mask to hardware
3646          * when modifying qp. If software need modify some fields in context,
3647          * we should set all bits of the relevant fields in context mask to
3648          * 0 at the same time, else set them to 0x1.
3649          */
3650         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3651                        V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3));
3652         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3653                        V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3654
3655         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3656                        V2_QPC_BYTE_12_SQ_HOP_NUM_S,
3657                        hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3658                        0 : hr_dev->caps.wqe_sq_hop_num);
3659         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3660                        V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3661
3662         roce_set_field(context->byte_20_smac_sgid_idx,
3663                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3664                        V2_QPC_BYTE_20_SGE_HOP_NUM_S,
3665                        ((ibqp->qp_type == IB_QPT_GSI) ||
3666                        hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3667                        hr_dev->caps.wqe_sge_hop_num : 0);
3668         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3669                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3670                        V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3671
3672         roce_set_field(context->byte_20_smac_sgid_idx,
3673                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3674                        V2_QPC_BYTE_20_RQ_HOP_NUM_S,
3675                        hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
3676                        0 : hr_dev->caps.wqe_rq_hop_num);
3677         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3678                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3679                        V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3680
3681         roce_set_field(context->byte_16_buf_ba_pg_sz,
3682                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3683                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
3684                        hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
3685         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3686                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3687                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3688
3689         roce_set_field(context->byte_16_buf_ba_pg_sz,
3690                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3691                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
3692                        hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
3693         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3694                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3695                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3696
3697         context->rq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3698         qpc_mask->rq_cur_blk_addr = 0;
3699
3700         roce_set_field(context->byte_92_srq_info,
3701                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3702                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
3703                        mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3704         roce_set_field(qpc_mask->byte_92_srq_info,
3705                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3706                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3707
3708         context->rq_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
3709         qpc_mask->rq_nxt_blk_addr = 0;
3710
3711         roce_set_field(context->byte_104_rq_sge,
3712                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3713                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
3714                        mtts[1] >> (32 + PAGE_ADDR_SHIFT));
3715         roce_set_field(qpc_mask->byte_104_rq_sge,
3716                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3717                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3718
3719         roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3720                        V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
3721         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3722                        V2_QPC_BYTE_132_TRRL_BA_S, 0);
3723         context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
3724         qpc_mask->trrl_ba = 0;
3725         roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3726                        V2_QPC_BYTE_140_TRRL_BA_S,
3727                        (u32)(dma_handle_3 >> (32 + 16 + 4)));
3728         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3729                        V2_QPC_BYTE_140_TRRL_BA_S, 0);
3730
3731         context->irrl_ba = (u32)(dma_handle_2 >> 6);
3732         qpc_mask->irrl_ba = 0;
3733         roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3734                        V2_QPC_BYTE_208_IRRL_BA_S,
3735                        dma_handle_2 >> (32 + 6));
3736         roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3737                        V2_QPC_BYTE_208_IRRL_BA_S, 0);
3738
3739         roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
3740         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
3741
3742         roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3743                      hr_qp->sq_signal_bits);
3744         roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3745                      0);
3746
3747         port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3748
3749         smac = (u8 *)hr_dev->dev_addr[port];
3750         /* when dmac equals smac or loop_idc is 1, it should loopback */
3751         if (ether_addr_equal_unaligned(dmac, smac) ||
3752             hr_dev->loop_idc == 0x1) {
3753                 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3754                 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3755         }
3756
3757         if (attr_mask & IB_QP_DEST_QPN) {
3758                 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3759                                V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
3760                 roce_set_field(qpc_mask->byte_56_dqpn_err,
3761                                V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3762         }
3763
3764         /* Configure GID index */
3765         port_num = rdma_ah_get_port_num(&attr->ah_attr);
3766         roce_set_field(context->byte_20_smac_sgid_idx,
3767                        V2_QPC_BYTE_20_SGID_IDX_M,
3768                        V2_QPC_BYTE_20_SGID_IDX_S,
3769                        hns_get_gid_index(hr_dev, port_num - 1,
3770                                          grh->sgid_index));
3771         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3772                        V2_QPC_BYTE_20_SGID_IDX_M,
3773                        V2_QPC_BYTE_20_SGID_IDX_S, 0);
3774         memcpy(&(context->dmac), dmac, sizeof(u32));
3775         roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3776                        V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
3777         qpc_mask->dmac = 0;
3778         roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3779                        V2_QPC_BYTE_52_DMAC_S, 0);
3780
3781         /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
3782         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3783                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
3784         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3785                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3786
3787         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
3788                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3789                                V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
3790         else if (attr_mask & IB_QP_PATH_MTU)
3791                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3792                                V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
3793
3794         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3795                        V2_QPC_BYTE_24_MTU_S, 0);
3796
3797         roce_set_field(context->byte_84_rq_ci_pi,
3798                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3799                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3800         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3801                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3802                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3803
3804         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3805                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3806                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3807         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3808                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3809         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3810                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3811         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3812                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3813                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3814
3815         context->rq_rnr_timer = 0;
3816         qpc_mask->rq_rnr_timer = 0;
3817
3818         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3819                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3820         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3821                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3822
3823         /* rocee send 2^lp_sgen_ini segs every time */
3824         roce_set_field(context->byte_168_irrl_idx,
3825                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
3826                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
3827         roce_set_field(qpc_mask->byte_168_irrl_idx,
3828                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
3829                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
3830
3831         return 0;
3832 }
3833
3834 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
3835                                 const struct ib_qp_attr *attr, int attr_mask,
3836                                 struct hns_roce_v2_qp_context *context,
3837                                 struct hns_roce_v2_qp_context *qpc_mask)
3838 {
3839         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3840         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3841         struct device *dev = hr_dev->dev;
3842         u64 sge_cur_blk = 0;
3843         u64 sq_cur_blk = 0;
3844         u32 page_size;
3845         int count;
3846
3847         /* Search qp buf's mtts */
3848         count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
3849         if (count < 1) {
3850                 dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn);
3851                 return -EINVAL;
3852         }
3853
3854         if (hr_qp->sge.offset) {
3855                 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3856                 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
3857                                           hr_qp->sge.offset / page_size,
3858                                           &sge_cur_blk, 1, NULL);
3859                 if (count < 1) {
3860                         dev_err(dev, "qp(0x%lx) sge pa find failed\n",
3861                                 hr_qp->qpn);
3862                         return -EINVAL;
3863                 }
3864         }
3865
3866         /* Not support alternate path and path migration */
3867         if ((attr_mask & IB_QP_ALT_PATH) ||
3868             (attr_mask & IB_QP_PATH_MIG_STATE)) {
3869                 dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3870                 return -EINVAL;
3871         }
3872
3873         /*
3874          * In v2 engine, software pass context and context mask to hardware
3875          * when modifying qp. If software need modify some fields in context,
3876          * we should set all bits of the relevant fields in context mask to
3877          * 0 at the same time, else set them to 0x1.
3878          */
3879         context->sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT);
3880         roce_set_field(context->byte_168_irrl_idx,
3881                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3882                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
3883                        sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
3884         qpc_mask->sq_cur_blk_addr = 0;
3885         roce_set_field(qpc_mask->byte_168_irrl_idx,
3886                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3887                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3888
3889         context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
3890                        hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3891                        ((u32)(sge_cur_blk >>
3892                        PAGE_ADDR_SHIFT)) : 0;
3893         roce_set_field(context->byte_184_irrl_idx,
3894                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3895                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
3896                        ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
3897                        HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
3898                        (sge_cur_blk >>
3899                        (32 + PAGE_ADDR_SHIFT)) : 0);
3900         qpc_mask->sq_cur_sge_blk_addr = 0;
3901         roce_set_field(qpc_mask->byte_184_irrl_idx,
3902                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3903                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3904
3905         context->rx_sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT);
3906         roce_set_field(context->byte_232_irrl_sge,
3907                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3908                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
3909                        sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
3910         qpc_mask->rx_sq_cur_blk_addr = 0;
3911         roce_set_field(qpc_mask->byte_232_irrl_sge,
3912                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3913                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
3914
3915         /*
3916          * Set some fields in context to zero, Because the default values
3917          * of all fields in context are zero, we need not set them to 0 again.
3918          * but we should set the relevant fields of context mask to 0.
3919          */
3920         roce_set_field(qpc_mask->byte_232_irrl_sge,
3921                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3922                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3923
3924         roce_set_field(qpc_mask->byte_240_irrl_tail,
3925                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
3926                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3927
3928         roce_set_field(qpc_mask->byte_248_ack_psn,
3929                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3930                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3931         roce_set_bit(qpc_mask->byte_248_ack_psn,
3932                      V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
3933         roce_set_field(qpc_mask->byte_248_ack_psn,
3934                        V2_QPC_BYTE_248_IRRL_PSN_M,
3935                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3936
3937         roce_set_field(qpc_mask->byte_240_irrl_tail,
3938                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3939                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3940
3941         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3942                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3943                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3944
3945         roce_set_bit(qpc_mask->byte_248_ack_psn,
3946                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3947
3948         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3949                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3950
3951         roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3952                        V2_QPC_BYTE_212_LSN_S, 0x100);
3953         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3954                        V2_QPC_BYTE_212_LSN_S, 0);
3955
3956         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3957                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3958
3959         return 0;
3960 }
3961
3962 static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,
3963                                              enum ib_qp_state new_state)
3964 {
3965
3966         if ((cur_state != IB_QPS_RESET &&
3967             (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||
3968             ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &&
3969             (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||
3970             (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS))
3971                 return true;
3972
3973         return false;
3974
3975 }
3976
3977 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
3978                                  const struct ib_qp_attr *attr,
3979                                  int attr_mask, enum ib_qp_state cur_state,
3980                                  enum ib_qp_state new_state)
3981 {
3982         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3983         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3984         struct hns_roce_v2_qp_context *context;
3985         struct hns_roce_v2_qp_context *qpc_mask;
3986         struct device *dev = hr_dev->dev;
3987         int ret = -EINVAL;
3988
3989         context = kcalloc(2, sizeof(*context), GFP_ATOMIC);
3990         if (!context)
3991                 return -ENOMEM;
3992
3993         qpc_mask = context + 1;
3994         /*
3995          * In v2 engine, software pass context and context mask to hardware
3996          * when modifying qp. If software need modify some fields in context,
3997          * we should set all bits of the relevant fields in context mask to
3998          * 0 at the same time, else set them to 0x1.
3999          */
4000         memset(qpc_mask, 0xff, sizeof(*qpc_mask));
4001         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4002                 memset(qpc_mask, 0, sizeof(*qpc_mask));
4003                 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
4004                                         qpc_mask);
4005         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4006                 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
4007                                        qpc_mask);
4008         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4009                 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
4010                                             qpc_mask);
4011                 if (ret)
4012                         goto out;
4013         } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4014                 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
4015                                            qpc_mask);
4016                 if (ret)
4017                         goto out;
4018         } else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {
4019                 /* Nothing */
4020                 ;
4021         } else {
4022                 dev_err(dev, "Illegal state for QP!\n");
4023                 ret = -EINVAL;
4024                 goto out;
4025         }
4026
4027         /* When QP state is err, SQ and RQ WQE should be flushed */
4028         if (new_state == IB_QPS_ERR) {
4029                 roce_set_field(context->byte_160_sq_ci_pi,
4030                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4031                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
4032                                hr_qp->sq.head);
4033                 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
4034                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
4035                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
4036
4037                 if (!ibqp->srq) {
4038                         roce_set_field(context->byte_84_rq_ci_pi,
4039                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4040                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
4041                                hr_qp->rq.head);
4042                         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
4043                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
4044                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
4045                 }
4046         }
4047
4048         if (attr_mask & IB_QP_AV) {
4049                 const struct ib_global_route *grh =
4050                                             rdma_ah_read_grh(&attr->ah_attr);
4051                 const struct ib_gid_attr *gid_attr = NULL;
4052                 int is_roce_protocol;
4053                 u16 vlan = 0xffff;
4054                 u8 ib_port;
4055                 u8 hr_port;
4056
4057                 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num :
4058                            hr_qp->port + 1;
4059                 hr_port = ib_port - 1;
4060                 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4061                                rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4062
4063                 if (is_roce_protocol) {
4064                         gid_attr = attr->ah_attr.grh.sgid_attr;
4065                         ret = rdma_read_gid_l2_fields(gid_attr, &vlan, NULL);
4066                         if (ret)
4067                                 goto out;
4068                 }
4069
4070                 if (vlan < VLAN_CFI_MASK) {
4071                         roce_set_bit(context->byte_76_srqn_op_en,
4072                                      V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
4073                         roce_set_bit(qpc_mask->byte_76_srqn_op_en,
4074                                      V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
4075                         roce_set_bit(context->byte_168_irrl_idx,
4076                                      V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
4077                         roce_set_bit(qpc_mask->byte_168_irrl_idx,
4078                                      V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
4079                 }
4080
4081                 roce_set_field(context->byte_24_mtu_tc,
4082                                V2_QPC_BYTE_24_VLAN_ID_M,
4083                                V2_QPC_BYTE_24_VLAN_ID_S, vlan);
4084                 roce_set_field(qpc_mask->byte_24_mtu_tc,
4085                                V2_QPC_BYTE_24_VLAN_ID_M,
4086                                V2_QPC_BYTE_24_VLAN_ID_S, 0);
4087
4088                 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
4089                         dev_err(hr_dev->dev,
4090                                 "sgid_index(%u) too large. max is %d\n",
4091                                 grh->sgid_index,
4092                                 hr_dev->caps.gid_table_len[hr_port]);
4093                         ret = -EINVAL;
4094                         goto out;
4095                 }
4096
4097                 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
4098                         dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
4099                         ret = -EINVAL;
4100                         goto out;
4101                 }
4102
4103                 roce_set_field(context->byte_52_udpspn_dmac,
4104                            V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S,
4105                            (gid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ?
4106                            0 : 0x12b7);
4107
4108                 roce_set_field(qpc_mask->byte_52_udpspn_dmac,
4109                                V2_QPC_BYTE_52_UDPSPN_M,
4110                                V2_QPC_BYTE_52_UDPSPN_S, 0);
4111
4112                 roce_set_field(context->byte_20_smac_sgid_idx,
4113                                V2_QPC_BYTE_20_SGID_IDX_M,
4114                                V2_QPC_BYTE_20_SGID_IDX_S, grh->sgid_index);
4115
4116                 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
4117                                V2_QPC_BYTE_20_SGID_IDX_M,
4118                                V2_QPC_BYTE_20_SGID_IDX_S, 0);
4119
4120                 roce_set_field(context->byte_24_mtu_tc,
4121                                V2_QPC_BYTE_24_HOP_LIMIT_M,
4122                                V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
4123                 roce_set_field(qpc_mask->byte_24_mtu_tc,
4124                                V2_QPC_BYTE_24_HOP_LIMIT_M,
4125                                V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
4126
4127                 if (hr_dev->pci_dev->revision == 0x21 &&
4128                     gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4129                         roce_set_field(context->byte_24_mtu_tc,
4130                                        V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,
4131                                        grh->traffic_class >> 2);
4132                 else
4133                         roce_set_field(context->byte_24_mtu_tc,
4134                                        V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,
4135                                        grh->traffic_class);
4136                 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
4137                                V2_QPC_BYTE_24_TC_S, 0);
4138                 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4139                                V2_QPC_BYTE_28_FL_S, grh->flow_label);
4140                 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
4141                                V2_QPC_BYTE_28_FL_S, 0);
4142                 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4143                 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
4144                 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4145                                V2_QPC_BYTE_28_SL_S,
4146                                rdma_ah_get_sl(&attr->ah_attr));
4147                 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
4148                                V2_QPC_BYTE_28_SL_S, 0);
4149                 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4150         }
4151
4152         if (attr_mask & IB_QP_TIMEOUT) {
4153                 if (attr->timeout < 31) {
4154                         roce_set_field(context->byte_28_at_fl,
4155                                        V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4156                                        attr->timeout);
4157                         roce_set_field(qpc_mask->byte_28_at_fl,
4158                                        V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
4159                                        0);
4160                 } else {
4161                         dev_warn(dev, "Local ACK timeout shall be 0 to 30.\n");
4162                 }
4163         }
4164
4165         if (attr_mask & IB_QP_RETRY_CNT) {
4166                 roce_set_field(context->byte_212_lsn,
4167                                V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4168                                V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
4169                                attr->retry_cnt);
4170                 roce_set_field(qpc_mask->byte_212_lsn,
4171                                V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
4172                                V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
4173
4174                 roce_set_field(context->byte_212_lsn,
4175                                V2_QPC_BYTE_212_RETRY_CNT_M,
4176                                V2_QPC_BYTE_212_RETRY_CNT_S,
4177                                attr->retry_cnt);
4178                 roce_set_field(qpc_mask->byte_212_lsn,
4179                                V2_QPC_BYTE_212_RETRY_CNT_M,
4180                                V2_QPC_BYTE_212_RETRY_CNT_S, 0);
4181         }
4182
4183         if (attr_mask & IB_QP_RNR_RETRY) {
4184                 roce_set_field(context->byte_244_rnr_rxack,
4185                                V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4186                                V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
4187                 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4188                                V2_QPC_BYTE_244_RNR_NUM_INIT_M,
4189                                V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
4190
4191                 roce_set_field(context->byte_244_rnr_rxack,
4192                                V2_QPC_BYTE_244_RNR_CNT_M,
4193                                V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
4194                 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4195                                V2_QPC_BYTE_244_RNR_CNT_M,
4196                                V2_QPC_BYTE_244_RNR_CNT_S, 0);
4197         }
4198
4199         if (attr_mask & IB_QP_SQ_PSN) {
4200                 roce_set_field(context->byte_172_sq_psn,
4201                                V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4202                                V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
4203                 roce_set_field(qpc_mask->byte_172_sq_psn,
4204                                V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4205                                V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
4206
4207                 roce_set_field(context->byte_196_sq_psn,
4208                                V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4209                                V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
4210                 roce_set_field(qpc_mask->byte_196_sq_psn,
4211                                V2_QPC_BYTE_196_SQ_MAX_PSN_M,
4212                                V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
4213
4214                 roce_set_field(context->byte_220_retry_psn_msn,
4215                                V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4216                                V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
4217                 roce_set_field(qpc_mask->byte_220_retry_psn_msn,
4218                                V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
4219                                V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
4220
4221                 roce_set_field(context->byte_224_retry_msg,
4222                                V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4223                                V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
4224                                attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S);
4225                 roce_set_field(qpc_mask->byte_224_retry_msg,
4226                                V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
4227                                V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
4228
4229                 roce_set_field(context->byte_224_retry_msg,
4230                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4231                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
4232                                attr->sq_psn);
4233                 roce_set_field(qpc_mask->byte_224_retry_msg,
4234                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
4235                                V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
4236
4237                 roce_set_field(context->byte_244_rnr_rxack,
4238                                V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4239                                V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
4240                 roce_set_field(qpc_mask->byte_244_rnr_rxack,
4241                                V2_QPC_BYTE_244_RX_ACK_EPSN_M,
4242                                V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
4243         }
4244
4245         if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
4246              attr->max_dest_rd_atomic) {
4247                 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4248                                V2_QPC_BYTE_140_RR_MAX_S,
4249                                fls(attr->max_dest_rd_atomic - 1));
4250                 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
4251                                V2_QPC_BYTE_140_RR_MAX_S, 0);
4252         }
4253
4254         if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
4255                 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
4256                                V2_QPC_BYTE_208_SR_MAX_S,
4257                                fls(attr->max_rd_atomic - 1));
4258                 roce_set_field(qpc_mask->byte_208_irrl,
4259                                V2_QPC_BYTE_208_SR_MAX_M,
4260                                V2_QPC_BYTE_208_SR_MAX_S, 0);
4261         }
4262
4263         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
4264                 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
4265
4266         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
4267                 roce_set_field(context->byte_80_rnr_rx_cqn,
4268                                V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4269                                V2_QPC_BYTE_80_MIN_RNR_TIME_S,
4270                                attr->min_rnr_timer);
4271                 roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
4272                                V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4273                                V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
4274         }
4275
4276         /* RC&UC required attr */
4277         if (attr_mask & IB_QP_RQ_PSN) {
4278                 roce_set_field(context->byte_108_rx_reqepsn,
4279                                V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4280                                V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
4281                 roce_set_field(qpc_mask->byte_108_rx_reqepsn,
4282                                V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4283                                V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
4284
4285                 roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
4286                                V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
4287                 roce_set_field(qpc_mask->byte_152_raq,
4288                                V2_QPC_BYTE_152_RAQ_PSN_M,
4289                                V2_QPC_BYTE_152_RAQ_PSN_S, 0);
4290         }
4291
4292         if (attr_mask & IB_QP_QKEY) {
4293                 context->qkey_xrcd = attr->qkey;
4294                 qpc_mask->qkey_xrcd = 0;
4295                 hr_qp->qkey = attr->qkey;
4296         }
4297
4298         roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
4299                      ibqp->srq ? 1 : 0);
4300         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
4301                      V2_QPC_BYTE_108_INV_CREDIT_S, 0);
4302
4303         /* Every status migrate must change state */
4304         roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4305                        V2_QPC_BYTE_60_QP_ST_S, new_state);
4306         roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
4307                        V2_QPC_BYTE_60_QP_ST_S, 0);
4308
4309         /* SW pass context to HW */
4310         ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state,
4311                                     context, hr_qp);
4312         if (ret) {
4313                 dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
4314                 goto out;
4315         }
4316
4317         hr_qp->state = new_state;
4318
4319         if (attr_mask & IB_QP_ACCESS_FLAGS)
4320                 hr_qp->atomic_rd_en = attr->qp_access_flags;
4321
4322         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4323                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
4324         if (attr_mask & IB_QP_PORT) {
4325                 hr_qp->port = attr->port_num - 1;
4326                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
4327         }
4328
4329         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
4330                 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
4331                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
4332                 if (ibqp->send_cq != ibqp->recv_cq)
4333                         hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
4334                                              hr_qp->qpn, NULL);
4335
4336                 hr_qp->rq.head = 0;
4337                 hr_qp->rq.tail = 0;
4338                 hr_qp->sq.head = 0;
4339                 hr_qp->sq.tail = 0;
4340                 hr_qp->sq_next_wqe = 0;
4341                 hr_qp->next_sge = 0;
4342                 if (hr_qp->rq.wqe_cnt)
4343                         *hr_qp->rdb.db_record = 0;
4344         }
4345
4346 out:
4347         kfree(context);
4348         return ret;
4349 }
4350
4351 static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
4352 {
4353         switch (state) {
4354         case HNS_ROCE_QP_ST_RST:        return IB_QPS_RESET;
4355         case HNS_ROCE_QP_ST_INIT:       return IB_QPS_INIT;
4356         case HNS_ROCE_QP_ST_RTR:        return IB_QPS_RTR;
4357         case HNS_ROCE_QP_ST_RTS:        return IB_QPS_RTS;
4358         case HNS_ROCE_QP_ST_SQ_DRAINING:
4359         case HNS_ROCE_QP_ST_SQD:        return IB_QPS_SQD;
4360         case HNS_ROCE_QP_ST_SQER:       return IB_QPS_SQE;
4361         case HNS_ROCE_QP_ST_ERR:        return IB_QPS_ERR;
4362         default:                        return -1;
4363         }
4364 }
4365
4366 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
4367                                  struct hns_roce_qp *hr_qp,
4368                                  struct hns_roce_v2_qp_context *hr_context)
4369 {
4370         struct hns_roce_cmd_mailbox *mailbox;
4371         int ret;
4372
4373         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4374         if (IS_ERR(mailbox))
4375                 return PTR_ERR(mailbox);
4376
4377         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
4378                                 HNS_ROCE_CMD_QUERY_QPC,
4379                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
4380         if (ret) {
4381                 dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
4382                 goto out;
4383         }
4384
4385         memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
4386
4387 out:
4388         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4389         return ret;
4390 }
4391
4392 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4393                                 int qp_attr_mask,
4394                                 struct ib_qp_init_attr *qp_init_attr)
4395 {
4396         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4397         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4398         struct hns_roce_v2_qp_context *context;
4399         struct device *dev = hr_dev->dev;
4400         int tmp_qp_state;
4401         int state;
4402         int ret;
4403
4404         context = kzalloc(sizeof(*context), GFP_KERNEL);
4405         if (!context)
4406                 return -ENOMEM;
4407
4408         memset(qp_attr, 0, sizeof(*qp_attr));
4409         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
4410
4411         mutex_lock(&hr_qp->mutex);
4412
4413         if (hr_qp->state == IB_QPS_RESET) {
4414                 qp_attr->qp_state = IB_QPS_RESET;
4415                 ret = 0;
4416                 goto done;
4417         }
4418
4419         ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context);
4420         if (ret) {
4421                 dev_err(dev, "query qpc error\n");
4422                 ret = -EINVAL;
4423                 goto out;
4424         }
4425
4426         state = roce_get_field(context->byte_60_qpst_tempid,
4427                                V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
4428         tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
4429         if (tmp_qp_state == -1) {
4430                 dev_err(dev, "Illegal ib_qp_state\n");
4431                 ret = -EINVAL;
4432                 goto out;
4433         }
4434         hr_qp->state = (u8)tmp_qp_state;
4435         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
4436         qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc,
4437                                                         V2_QPC_BYTE_24_MTU_M,
4438                                                         V2_QPC_BYTE_24_MTU_S);
4439         qp_attr->path_mig_state = IB_MIG_ARMED;
4440         qp_attr->ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;
4441         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
4442                 qp_attr->qkey = V2_QKEY_VAL;
4443
4444         qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn,
4445                                          V2_QPC_BYTE_108_RX_REQ_EPSN_M,
4446                                          V2_QPC_BYTE_108_RX_REQ_EPSN_S);
4447         qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn,
4448                                               V2_QPC_BYTE_172_SQ_CUR_PSN_M,
4449                                               V2_QPC_BYTE_172_SQ_CUR_PSN_S);
4450         qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err,
4451                                                   V2_QPC_BYTE_56_DQPN_M,
4452                                                   V2_QPC_BYTE_56_DQPN_S);
4453         qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en,
4454                                     V2_QPC_BYTE_76_RRE_S)) << V2_QP_RWE_S) |
4455                                     ((roce_get_bit(context->byte_76_srqn_op_en,
4456                                     V2_QPC_BYTE_76_RWE_S)) << V2_QP_RRE_S) |
4457                                     ((roce_get_bit(context->byte_76_srqn_op_en,
4458                                     V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S);
4459
4460         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
4461             hr_qp->ibqp.qp_type == IB_QPT_UC) {
4462                 struct ib_global_route *grh =
4463                                 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
4464
4465                 rdma_ah_set_sl(&qp_attr->ah_attr,
4466                                roce_get_field(context->byte_28_at_fl,
4467                                               V2_QPC_BYTE_28_SL_M,
4468                                               V2_QPC_BYTE_28_SL_S));
4469                 grh->flow_label = roce_get_field(context->byte_28_at_fl,
4470                                                  V2_QPC_BYTE_28_FL_M,
4471                                                  V2_QPC_BYTE_28_FL_S);
4472                 grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx,
4473                                                  V2_QPC_BYTE_20_SGID_IDX_M,
4474                                                  V2_QPC_BYTE_20_SGID_IDX_S);
4475                 grh->hop_limit = roce_get_field(context->byte_24_mtu_tc,
4476                                                 V2_QPC_BYTE_24_HOP_LIMIT_M,
4477                                                 V2_QPC_BYTE_24_HOP_LIMIT_S);
4478                 grh->traffic_class = roce_get_field(context->byte_24_mtu_tc,
4479                                                     V2_QPC_BYTE_24_TC_M,
4480                                                     V2_QPC_BYTE_24_TC_S);
4481
4482                 memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw));
4483         }
4484
4485         qp_attr->port_num = hr_qp->port + 1;
4486         qp_attr->sq_draining = 0;
4487         qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl,
4488                                                      V2_QPC_BYTE_208_SR_MAX_M,
4489                                                      V2_QPC_BYTE_208_SR_MAX_S);
4490         qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq,
4491                                                      V2_QPC_BYTE_140_RR_MAX_M,
4492                                                      V2_QPC_BYTE_140_RR_MAX_S);
4493         qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn,
4494                                                  V2_QPC_BYTE_80_MIN_RNR_TIME_M,
4495                                                  V2_QPC_BYTE_80_MIN_RNR_TIME_S);
4496         qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl,
4497                                               V2_QPC_BYTE_28_AT_M,
4498                                               V2_QPC_BYTE_28_AT_S);
4499         qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn,
4500                                             V2_QPC_BYTE_212_RETRY_CNT_M,
4501                                             V2_QPC_BYTE_212_RETRY_CNT_S);
4502         qp_attr->rnr_retry = context->rq_rnr_timer;
4503
4504 done:
4505         qp_attr->cur_qp_state = qp_attr->qp_state;
4506         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
4507         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
4508
4509         if (!ibqp->uobject) {
4510                 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
4511                 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
4512         } else {
4513                 qp_attr->cap.max_send_wr = 0;
4514                 qp_attr->cap.max_send_sge = 0;
4515         }
4516
4517         qp_init_attr->cap = qp_attr->cap;
4518
4519 out:
4520         mutex_unlock(&hr_qp->mutex);
4521         kfree(context);
4522         return ret;
4523 }
4524
4525 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
4526                                          struct hns_roce_qp *hr_qp,
4527                                          struct ib_udata *udata)
4528 {
4529         struct hns_roce_cq *send_cq, *recv_cq;
4530         struct device *dev = hr_dev->dev;
4531         int ret;
4532
4533         if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
4534                 /* Modify qp to reset before destroying qp */
4535                 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
4536                                             hr_qp->state, IB_QPS_RESET);
4537                 if (ret) {
4538                         dev_err(dev, "modify QP %06lx to ERR failed.\n",
4539                                 hr_qp->qpn);
4540                         return ret;
4541                 }
4542         }
4543
4544         send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
4545         recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
4546
4547         hns_roce_lock_cqs(send_cq, recv_cq);
4548
4549         if (!udata) {
4550                 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
4551                                        to_hr_srq(hr_qp->ibqp.srq) : NULL);
4552                 if (send_cq != recv_cq)
4553                         __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
4554         }
4555
4556         hns_roce_qp_remove(hr_dev, hr_qp);
4557
4558         hns_roce_unlock_cqs(send_cq, recv_cq);
4559
4560         hns_roce_qp_free(hr_dev, hr_qp);
4561
4562         /* Not special_QP, free their QPN */
4563         if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
4564             (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
4565             (hr_qp->ibqp.qp_type == IB_QPT_UD))
4566                 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
4567
4568         hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
4569
4570         if (udata) {
4571                 struct hns_roce_ucontext *context =
4572                         rdma_udata_to_drv_context(
4573                                 udata,
4574                                 struct hns_roce_ucontext,
4575                                 ibucontext);
4576
4577                 if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
4578                         hns_roce_db_unmap_user(context, &hr_qp->sdb);
4579
4580                 if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
4581                         hns_roce_db_unmap_user(context, &hr_qp->rdb);
4582         } else {
4583                 kfree(hr_qp->sq.wrid);
4584                 kfree(hr_qp->rq.wrid);
4585                 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
4586                 if (hr_qp->rq.wqe_cnt)
4587                         hns_roce_free_db(hr_dev, &hr_qp->rdb);
4588         }
4589         ib_umem_release(hr_qp->umem);
4590
4591         if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
4592              hr_qp->rq.wqe_cnt) {
4593                 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4594                 kfree(hr_qp->rq_inl_buf.wqe_list);
4595         }
4596
4597         return 0;
4598 }
4599
4600 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
4601 {
4602         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4603         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4604         int ret;
4605
4606         ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
4607         if (ret) {
4608                 dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret);
4609                 return ret;
4610         }
4611
4612         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
4613                 kfree(hr_to_hr_sqp(hr_qp));
4614         else
4615                 kfree(hr_qp);
4616
4617         return 0;
4618 }
4619
4620 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
4621                                                 struct hns_roce_qp *hr_qp)
4622 {
4623         struct hns_roce_sccc_clr_done *resp;
4624         struct hns_roce_sccc_clr *clr;
4625         struct hns_roce_cmq_desc desc;
4626         int ret, i;
4627
4628         mutex_lock(&hr_dev->qp_table.scc_mutex);
4629
4630         /* set scc ctx clear done flag */
4631         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
4632         ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
4633         if (ret) {
4634                 dev_err(hr_dev->dev, "Reset SCC ctx  failed(%d)\n", ret);
4635                 goto out;
4636         }
4637
4638         /* clear scc context */
4639         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
4640         clr = (struct hns_roce_sccc_clr *)desc.data;
4641         clr->qpn = cpu_to_le32(hr_qp->qpn);
4642         ret =  hns_roce_cmq_send(hr_dev, &desc, 1);
4643         if (ret) {
4644                 dev_err(hr_dev->dev, "Clear SCC ctx failed(%d)\n", ret);
4645                 goto out;
4646         }
4647
4648         /* query scc context clear is done or not */
4649         resp = (struct hns_roce_sccc_clr_done *)desc.data;
4650         for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
4651                 hns_roce_cmq_setup_basic_desc(&desc,
4652                                               HNS_ROCE_OPC_QUERY_SCCC, true);
4653                 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
4654                 if (ret) {
4655                         dev_err(hr_dev->dev, "Query clr cmq failed(%d)\n", ret);
4656                         goto out;
4657                 }
4658
4659                 if (resp->clr_done)
4660                         goto out;
4661
4662                 msleep(20);
4663         }
4664
4665         dev_err(hr_dev->dev, "Query SCC clr done flag overtime.\n");
4666         ret = -ETIMEDOUT;
4667
4668 out:
4669         mutex_unlock(&hr_dev->qp_table.scc_mutex);
4670         return ret;
4671 }
4672
4673 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
4674 {
4675         struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
4676         struct hns_roce_v2_cq_context *cq_context;
4677         struct hns_roce_cq *hr_cq = to_hr_cq(cq);
4678         struct hns_roce_v2_cq_context *cqc_mask;
4679         struct hns_roce_cmd_mailbox *mailbox;
4680         int ret;
4681
4682         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4683         if (IS_ERR(mailbox))
4684                 return PTR_ERR(mailbox);
4685
4686         cq_context = mailbox->buf;
4687         cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
4688
4689         memset(cqc_mask, 0xff, sizeof(*cqc_mask));
4690
4691         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4692                        V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4693                        cq_count);
4694         roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4695                        V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4696                        0);
4697         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4698                        V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4699                        cq_period);
4700         roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4701                        V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4702                        0);
4703
4704         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
4705                                 HNS_ROCE_CMD_MODIFY_CQC,
4706                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
4707         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4708         if (ret)
4709                 dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
4710
4711         return ret;
4712 }
4713
4714 static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
4715 {
4716         struct hns_roce_qp *hr_qp;
4717         struct ib_qp_attr attr;
4718         int attr_mask;
4719         int ret;
4720
4721         hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
4722         if (!hr_qp) {
4723                 dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
4724                 return;
4725         }
4726
4727         if (hr_qp->ibqp.uobject) {
4728                 if (hr_qp->sdb_en == 1) {
4729                         hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
4730                         if (hr_qp->rdb_en == 1)
4731                                 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
4732                 } else {
4733                         dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
4734                         return;
4735                 }
4736         }
4737
4738         attr_mask = IB_QP_STATE;
4739         attr.qp_state = IB_QPS_ERR;
4740         ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
4741                                     hr_qp->state, IB_QPS_ERR);
4742         if (ret)
4743                 dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
4744                         qpn);
4745 }
4746
4747 static void hns_roce_irq_work_handle(struct work_struct *work)
4748 {
4749         struct hns_roce_work *irq_work =
4750                                 container_of(work, struct hns_roce_work, work);
4751         struct device *dev = irq_work->hr_dev->dev;
4752         u32 qpn = irq_work->qpn;
4753         u32 cqn = irq_work->cqn;
4754
4755         switch (irq_work->event_type) {
4756         case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4757                 dev_info(dev, "Path migrated succeeded.\n");
4758                 break;
4759         case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4760                 dev_warn(dev, "Path migration failed.\n");
4761                 break;
4762         case HNS_ROCE_EVENT_TYPE_COMM_EST:
4763                 break;
4764         case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4765                 dev_warn(dev, "Send queue drained.\n");
4766                 break;
4767         case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4768                 dev_err(dev, "Local work queue 0x%x catas error, sub_type:%d\n",
4769                         qpn, irq_work->sub_type);
4770                 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4771                 break;
4772         case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4773                 dev_err(dev, "Invalid request local work queue 0x%x error.\n",
4774                         qpn);
4775                 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4776                 break;
4777         case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4778                 dev_err(dev, "Local access violation work queue 0x%x error, sub_type:%d\n",
4779                         qpn, irq_work->sub_type);
4780                 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4781                 break;
4782         case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4783                 dev_warn(dev, "SRQ limit reach.\n");
4784                 break;
4785         case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4786                 dev_warn(dev, "SRQ last wqe reach.\n");
4787                 break;
4788         case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4789                 dev_err(dev, "SRQ catas error.\n");
4790                 break;
4791         case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4792                 dev_err(dev, "CQ 0x%x access err.\n", cqn);
4793                 break;
4794         case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4795                 dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4796                 break;
4797         case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4798                 dev_warn(dev, "DB overflow.\n");
4799                 break;
4800         case HNS_ROCE_EVENT_TYPE_FLR:
4801                 dev_warn(dev, "Function level reset.\n");
4802                 break;
4803         default:
4804                 break;
4805         }
4806
4807         kfree(irq_work);
4808 }
4809
4810 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
4811                                       struct hns_roce_eq *eq,
4812                                       u32 qpn, u32 cqn)
4813 {
4814         struct hns_roce_work *irq_work;
4815
4816         irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
4817         if (!irq_work)
4818                 return;
4819
4820         INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4821         irq_work->hr_dev = hr_dev;
4822         irq_work->qpn = qpn;
4823         irq_work->cqn = cqn;
4824         irq_work->event_type = eq->event_type;
4825         irq_work->sub_type = eq->sub_type;
4826         queue_work(hr_dev->irq_workq, &(irq_work->work));
4827 }
4828
4829 static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4830 {
4831         struct hns_roce_dev *hr_dev = eq->hr_dev;
4832         u32 doorbell[2];
4833
4834         doorbell[0] = 0;
4835         doorbell[1] = 0;
4836
4837         if (eq->type_flag == HNS_ROCE_AEQ) {
4838                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4839                                HNS_ROCE_V2_EQ_DB_CMD_S,
4840                                eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4841                                HNS_ROCE_EQ_DB_CMD_AEQ :
4842                                HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
4843         } else {
4844                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
4845                                HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
4846
4847                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4848                                HNS_ROCE_V2_EQ_DB_CMD_S,
4849                                eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4850                                HNS_ROCE_EQ_DB_CMD_CEQ :
4851                                HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
4852         }
4853
4854         roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
4855                        HNS_ROCE_V2_EQ_DB_PARA_S,
4856                        (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4857
4858         hns_roce_write64(hr_dev, doorbell, eq->doorbell);
4859 }
4860
4861 static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
4862 {
4863         u32 buf_chk_sz;
4864         unsigned long off;
4865
4866         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4867         off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4868
4869         return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
4870                 off % buf_chk_sz);
4871 }
4872
4873 static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
4874 {
4875         u32 buf_chk_sz;
4876         unsigned long off;
4877
4878         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4879
4880         off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4881
4882         if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4883                 return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
4884                         off % buf_chk_sz);
4885         else
4886                 return (struct hns_roce_aeqe *)((u8 *)
4887                         (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
4888 }
4889
4890 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
4891 {
4892         struct hns_roce_aeqe *aeqe;
4893
4894         if (!eq->hop_num)
4895                 aeqe = get_aeqe_v2(eq, eq->cons_index);
4896         else
4897                 aeqe = mhop_get_aeqe(eq, eq->cons_index);
4898
4899         return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
4900                 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
4901 }
4902
4903 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
4904                                struct hns_roce_eq *eq)
4905 {
4906         struct device *dev = hr_dev->dev;
4907         struct hns_roce_aeqe *aeqe;
4908         int aeqe_found = 0;
4909         int event_type;
4910         int sub_type;
4911         u32 srqn;
4912         u32 qpn;
4913         u32 cqn;
4914
4915         while ((aeqe = next_aeqe_sw_v2(eq))) {
4916
4917                 /* Make sure we read AEQ entry after we have checked the
4918                  * ownership bit
4919                  */
4920                 dma_rmb();
4921
4922                 event_type = roce_get_field(aeqe->asyn,
4923                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
4924                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
4925                 sub_type = roce_get_field(aeqe->asyn,
4926                                           HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4927                                           HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4928                 qpn = roce_get_field(aeqe->event.qp_event.qp,
4929                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4930                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4931                 cqn = roce_get_field(aeqe->event.cq_event.cq,
4932                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4933                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4934                 srqn = roce_get_field(aeqe->event.srq_event.srq,
4935                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4936                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4937
4938                 switch (event_type) {
4939                 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4940                 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4941                 case HNS_ROCE_EVENT_TYPE_COMM_EST:
4942                 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4943                 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4944                 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4945                 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4946                 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4947                         hns_roce_qp_event(hr_dev, qpn, event_type);
4948                         break;
4949                 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4950                 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4951                         hns_roce_srq_event(hr_dev, srqn, event_type);
4952                         break;
4953                 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4954                 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4955                         hns_roce_cq_event(hr_dev, cqn, event_type);
4956                         break;
4957                 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4958                         break;
4959                 case HNS_ROCE_EVENT_TYPE_MB:
4960                         hns_roce_cmd_event(hr_dev,
4961                                         le16_to_cpu(aeqe->event.cmd.token),
4962                                         aeqe->event.cmd.status,
4963                                         le64_to_cpu(aeqe->event.cmd.out_param));
4964                         break;
4965                 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
4966                         break;
4967                 case HNS_ROCE_EVENT_TYPE_FLR:
4968                         break;
4969                 default:
4970                         dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
4971                                 event_type, eq->eqn, eq->cons_index);
4972                         break;
4973                 }
4974
4975                 eq->event_type = event_type;
4976                 eq->sub_type = sub_type;
4977                 ++eq->cons_index;
4978                 aeqe_found = 1;
4979
4980                 if (eq->cons_index > (2 * eq->entries - 1)) {
4981                         dev_warn(dev, "cons_index overflow, set back to 0.\n");
4982                         eq->cons_index = 0;
4983                 }
4984                 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
4985         }
4986
4987         set_eq_cons_index_v2(eq);
4988         return aeqe_found;
4989 }
4990
4991 static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
4992 {
4993         u32 buf_chk_sz;
4994         unsigned long off;
4995
4996         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4997         off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4998
4999         return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
5000                 off % buf_chk_sz);
5001 }
5002
5003 static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
5004 {
5005         u32 buf_chk_sz;
5006         unsigned long off;
5007
5008         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5009
5010         off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
5011
5012         if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
5013                 return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
5014                         off % buf_chk_sz);
5015         else
5016                 return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
5017                         buf_chk_sz]) + off % buf_chk_sz);
5018 }
5019
5020 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
5021 {
5022         struct hns_roce_ceqe *ceqe;
5023
5024         if (!eq->hop_num)
5025                 ceqe = get_ceqe_v2(eq, eq->cons_index);
5026         else
5027                 ceqe = mhop_get_ceqe(eq, eq->cons_index);
5028
5029         return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
5030                 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
5031 }
5032
5033 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
5034                                struct hns_roce_eq *eq)
5035 {
5036         struct device *dev = hr_dev->dev;
5037         struct hns_roce_ceqe *ceqe;
5038         int ceqe_found = 0;
5039         u32 cqn;
5040
5041         while ((ceqe = next_ceqe_sw_v2(eq))) {
5042
5043                 /* Make sure we read CEQ entry after we have checked the
5044                  * ownership bit
5045                  */
5046                 dma_rmb();
5047
5048                 cqn = roce_get_field(ceqe->comp,
5049                                      HNS_ROCE_V2_CEQE_COMP_CQN_M,
5050                                      HNS_ROCE_V2_CEQE_COMP_CQN_S);
5051
5052                 hns_roce_cq_completion(hr_dev, cqn);
5053
5054                 ++eq->cons_index;
5055                 ceqe_found = 1;
5056
5057                 if (eq->cons_index > (2 * eq->entries - 1)) {
5058                         dev_warn(dev, "cons_index overflow, set back to 0.\n");
5059                         eq->cons_index = 0;
5060                 }
5061         }
5062
5063         set_eq_cons_index_v2(eq);
5064
5065         return ceqe_found;
5066 }
5067
5068 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
5069 {
5070         struct hns_roce_eq *eq = eq_ptr;
5071         struct hns_roce_dev *hr_dev = eq->hr_dev;
5072         int int_work = 0;
5073
5074         if (eq->type_flag == HNS_ROCE_CEQ)
5075                 /* Completion event interrupt */
5076                 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
5077         else
5078                 /* Asychronous event interrupt */
5079                 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
5080
5081         return IRQ_RETVAL(int_work);
5082 }
5083
5084 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
5085 {
5086         struct hns_roce_dev *hr_dev = dev_id;
5087         struct device *dev = hr_dev->dev;
5088         int int_work = 0;
5089         u32 int_st;
5090         u32 int_en;
5091
5092         /* Abnormal interrupt */
5093         int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
5094         int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
5095
5096         if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
5097                 struct pci_dev *pdev = hr_dev->pci_dev;
5098                 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
5099                 const struct hnae3_ae_ops *ops = ae_dev->ops;
5100
5101                 dev_err(dev, "AEQ overflow!\n");
5102
5103                 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);
5104                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5105
5106                 /* Set reset level for reset_event() */
5107                 if (ops->set_default_reset_request)
5108                         ops->set_default_reset_request(ae_dev,
5109                                                        HNAE3_FUNC_RESET);
5110                 if (ops->reset_event)
5111                         ops->reset_event(pdev, NULL);
5112
5113                 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
5114                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5115
5116                 int_work = 1;
5117         } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
5118                 dev_err(dev, "BUS ERR!\n");
5119
5120                 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);
5121                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5122
5123                 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
5124                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5125
5126                 int_work = 1;
5127         } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
5128                 dev_err(dev, "OTHER ERR!\n");
5129
5130                 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);
5131                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
5132
5133                 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
5134                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
5135
5136                 int_work = 1;
5137         } else
5138                 dev_err(dev, "There is no abnormal irq found!\n");
5139
5140         return IRQ_RETVAL(int_work);
5141 }
5142
5143 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
5144                                         int eq_num, int enable_flag)
5145 {
5146         int i;
5147
5148         if (enable_flag == EQ_ENABLE) {
5149                 for (i = 0; i < eq_num; i++)
5150                         roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5151                                    i * EQ_REG_OFFSET,
5152                                    HNS_ROCE_V2_VF_EVENT_INT_EN_M);
5153
5154                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5155                            HNS_ROCE_V2_VF_ABN_INT_EN_M);
5156                 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5157                            HNS_ROCE_V2_VF_ABN_INT_CFG_M);
5158         } else {
5159                 for (i = 0; i < eq_num; i++)
5160                         roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
5161                                    i * EQ_REG_OFFSET,
5162                                    HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
5163
5164                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
5165                            HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
5166                 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
5167                            HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
5168         }
5169 }
5170
5171 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
5172 {
5173         struct device *dev = hr_dev->dev;
5174         int ret;
5175
5176         if (eqn < hr_dev->caps.num_comp_vectors)
5177                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5178                                         0, HNS_ROCE_CMD_DESTROY_CEQC,
5179                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5180         else
5181                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
5182                                         0, HNS_ROCE_CMD_DESTROY_AEQC,
5183                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
5184         if (ret)
5185                 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
5186 }
5187
5188 static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
5189                                   struct hns_roce_eq *eq)
5190 {
5191         struct device *dev = hr_dev->dev;
5192         u64 idx;
5193         u64 size;
5194         u32 buf_chk_sz;
5195         u32 bt_chk_sz;
5196         u32 mhop_num;
5197         int eqe_alloc;
5198         int i = 0;
5199         int j = 0;
5200
5201         mhop_num = hr_dev->caps.eqe_hop_num;
5202         buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5203         bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5204
5205         /* hop_num = 0 */
5206         if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5207                 dma_free_coherent(dev, (unsigned int)(eq->entries *
5208                                   eq->eqe_size), eq->bt_l0, eq->l0_dma);
5209                 return;
5210         }
5211
5212         /* hop_num = 1 or hop = 2 */
5213         dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5214         if (mhop_num == 1) {
5215                 for (i = 0; i < eq->l0_last_num; i++) {
5216                         if (i == eq->l0_last_num - 1) {
5217                                 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5218                                 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5219                                 dma_free_coherent(dev, size, eq->buf[i],
5220                                                   eq->buf_dma[i]);
5221                                 break;
5222                         }
5223                         dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5224                                           eq->buf_dma[i]);
5225                 }
5226         } else if (mhop_num == 2) {
5227                 for (i = 0; i < eq->l0_last_num; i++) {
5228                         dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5229                                           eq->l1_dma[i]);
5230
5231                         for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5232                                 idx = i * (bt_chk_sz / BA_BYTE_LEN) + j;
5233                                 if ((i == eq->l0_last_num - 1)
5234                                      && j == eq->l1_last_num - 1) {
5235                                         eqe_alloc = (buf_chk_sz / eq->eqe_size)
5236                                                     * idx;
5237                                         size = (eq->entries - eqe_alloc)
5238                                                 * eq->eqe_size;
5239                                         dma_free_coherent(dev, size,
5240                                                           eq->buf[idx],
5241                                                           eq->buf_dma[idx]);
5242                                         break;
5243                                 }
5244                                 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5245                                                   eq->buf_dma[idx]);
5246                         }
5247                 }
5248         }
5249         kfree(eq->buf_dma);
5250         kfree(eq->buf);
5251         kfree(eq->l1_dma);
5252         kfree(eq->bt_l1);
5253         eq->buf_dma = NULL;
5254         eq->buf = NULL;
5255         eq->l1_dma = NULL;
5256         eq->bt_l1 = NULL;
5257 }
5258
5259 static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
5260                                 struct hns_roce_eq *eq)
5261 {
5262         u32 buf_chk_sz;
5263
5264         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
5265
5266         if (hr_dev->caps.eqe_hop_num) {
5267                 hns_roce_mhop_free_eq(hr_dev, eq);
5268                 return;
5269         }
5270
5271         dma_free_coherent(hr_dev->dev, buf_chk_sz, eq->buf_list->buf,
5272                           eq->buf_list->map);
5273         kfree(eq->buf_list);
5274 }
5275
5276 static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
5277                                 struct hns_roce_eq *eq,
5278                                 void *mb_buf)
5279 {
5280         struct hns_roce_eq_context *eqc;
5281
5282         eqc = mb_buf;
5283         memset(eqc, 0, sizeof(struct hns_roce_eq_context));
5284
5285         /* init eqc */
5286         eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
5287         eq->hop_num = hr_dev->caps.eqe_hop_num;
5288         eq->cons_index = 0;
5289         eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
5290         eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
5291         eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
5292         eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
5293         eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
5294         eq->shift = ilog2((unsigned int)eq->entries);
5295
5296         if (!eq->hop_num)
5297                 eq->eqe_ba = eq->buf_list->map;
5298         else
5299                 eq->eqe_ba = eq->l0_dma;
5300
5301         /* set eqc state */
5302         roce_set_field(eqc->byte_4,
5303                        HNS_ROCE_EQC_EQ_ST_M,
5304                        HNS_ROCE_EQC_EQ_ST_S,
5305                        HNS_ROCE_V2_EQ_STATE_VALID);
5306
5307         /* set eqe hop num */
5308         roce_set_field(eqc->byte_4,
5309                        HNS_ROCE_EQC_HOP_NUM_M,
5310                        HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
5311
5312         /* set eqc over_ignore */
5313         roce_set_field(eqc->byte_4,
5314                        HNS_ROCE_EQC_OVER_IGNORE_M,
5315                        HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
5316
5317         /* set eqc coalesce */
5318         roce_set_field(eqc->byte_4,
5319                        HNS_ROCE_EQC_COALESCE_M,
5320                        HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
5321
5322         /* set eqc arm_state */
5323         roce_set_field(eqc->byte_4,
5324                        HNS_ROCE_EQC_ARM_ST_M,
5325                        HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
5326
5327         /* set eqn */
5328         roce_set_field(eqc->byte_4,
5329                        HNS_ROCE_EQC_EQN_M,
5330                        HNS_ROCE_EQC_EQN_S, eq->eqn);
5331
5332         /* set eqe_cnt */
5333         roce_set_field(eqc->byte_4,
5334                        HNS_ROCE_EQC_EQE_CNT_M,
5335                        HNS_ROCE_EQC_EQE_CNT_S,
5336                        HNS_ROCE_EQ_INIT_EQE_CNT);
5337
5338         /* set eqe_ba_pg_sz */
5339         roce_set_field(eqc->byte_8,
5340                        HNS_ROCE_EQC_BA_PG_SZ_M,
5341                        HNS_ROCE_EQC_BA_PG_SZ_S,
5342                        eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
5343
5344         /* set eqe_buf_pg_sz */
5345         roce_set_field(eqc->byte_8,
5346                        HNS_ROCE_EQC_BUF_PG_SZ_M,
5347                        HNS_ROCE_EQC_BUF_PG_SZ_S,
5348                        eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
5349
5350         /* set eq_producer_idx */
5351         roce_set_field(eqc->byte_8,
5352                        HNS_ROCE_EQC_PROD_INDX_M,
5353                        HNS_ROCE_EQC_PROD_INDX_S,
5354                        HNS_ROCE_EQ_INIT_PROD_IDX);
5355
5356         /* set eq_max_cnt */
5357         roce_set_field(eqc->byte_12,
5358                        HNS_ROCE_EQC_MAX_CNT_M,
5359                        HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
5360
5361         /* set eq_period */
5362         roce_set_field(eqc->byte_12,
5363                        HNS_ROCE_EQC_PERIOD_M,
5364                        HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
5365
5366         /* set eqe_report_timer */
5367         roce_set_field(eqc->eqe_report_timer,
5368                        HNS_ROCE_EQC_REPORT_TIMER_M,
5369                        HNS_ROCE_EQC_REPORT_TIMER_S,
5370                        HNS_ROCE_EQ_INIT_REPORT_TIMER);
5371
5372         /* set eqe_ba [34:3] */
5373         roce_set_field(eqc->eqe_ba0,
5374                        HNS_ROCE_EQC_EQE_BA_L_M,
5375                        HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
5376
5377         /* set eqe_ba [64:35] */
5378         roce_set_field(eqc->eqe_ba1,
5379                        HNS_ROCE_EQC_EQE_BA_H_M,
5380                        HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
5381
5382         /* set eq shift */
5383         roce_set_field(eqc->byte_28,
5384                        HNS_ROCE_EQC_SHIFT_M,
5385                        HNS_ROCE_EQC_SHIFT_S, eq->shift);
5386
5387         /* set eq MSI_IDX */
5388         roce_set_field(eqc->byte_28,
5389                        HNS_ROCE_EQC_MSI_INDX_M,
5390                        HNS_ROCE_EQC_MSI_INDX_S,
5391                        HNS_ROCE_EQ_INIT_MSI_IDX);
5392
5393         /* set cur_eqe_ba [27:12] */
5394         roce_set_field(eqc->byte_28,
5395                        HNS_ROCE_EQC_CUR_EQE_BA_L_M,
5396                        HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
5397
5398         /* set cur_eqe_ba [59:28] */
5399         roce_set_field(eqc->byte_32,
5400                        HNS_ROCE_EQC_CUR_EQE_BA_M_M,
5401                        HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
5402
5403         /* set cur_eqe_ba [63:60] */
5404         roce_set_field(eqc->byte_36,
5405                        HNS_ROCE_EQC_CUR_EQE_BA_H_M,
5406                        HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
5407
5408         /* set eq consumer idx */
5409         roce_set_field(eqc->byte_36,
5410                        HNS_ROCE_EQC_CONS_INDX_M,
5411                        HNS_ROCE_EQC_CONS_INDX_S,
5412                        HNS_ROCE_EQ_INIT_CONS_IDX);
5413
5414         /* set nex_eqe_ba[43:12] */
5415         roce_set_field(eqc->nxt_eqe_ba0,
5416                        HNS_ROCE_EQC_NXT_EQE_BA_L_M,
5417                        HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
5418
5419         /* set nex_eqe_ba[63:44] */
5420         roce_set_field(eqc->nxt_eqe_ba1,
5421                        HNS_ROCE_EQC_NXT_EQE_BA_H_M,
5422                        HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
5423 }
5424
5425 static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
5426                                   struct hns_roce_eq *eq)
5427 {
5428         struct device *dev = hr_dev->dev;
5429         int eq_alloc_done = 0;
5430         int eq_buf_cnt = 0;
5431         int eqe_alloc;
5432         u32 buf_chk_sz;
5433         u32 bt_chk_sz;
5434         u32 mhop_num;
5435         u64 size;
5436         u64 idx;
5437         int ba_num;
5438         int bt_num;
5439         int record_i;
5440         int record_j;
5441         int i = 0;
5442         int j = 0;
5443
5444         mhop_num = hr_dev->caps.eqe_hop_num;
5445         buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5446         bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
5447
5448         ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
5449                               buf_chk_sz);
5450         bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN);
5451
5452         /* hop_num = 0 */
5453         if (mhop_num == HNS_ROCE_HOP_NUM_0) {
5454                 if (eq->entries > buf_chk_sz / eq->eqe_size) {
5455                         dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
5456                                 eq->entries);
5457                         return -EINVAL;
5458                 }
5459                 eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
5460                                                &(eq->l0_dma), GFP_KERNEL);
5461                 if (!eq->bt_l0)
5462                         return -ENOMEM;
5463
5464                 eq->cur_eqe_ba = eq->l0_dma;
5465                 eq->nxt_eqe_ba = 0;
5466
5467                 return 0;
5468         }
5469
5470         eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
5471         if (!eq->buf_dma)
5472                 return -ENOMEM;
5473         eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
5474         if (!eq->buf)
5475                 goto err_kcalloc_buf;
5476
5477         if (mhop_num == 2) {
5478                 eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
5479                 if (!eq->l1_dma)
5480                         goto err_kcalloc_l1_dma;
5481
5482                 eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
5483                 if (!eq->bt_l1)
5484                         goto err_kcalloc_bt_l1;
5485         }
5486
5487         /* alloc L0 BT */
5488         eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
5489         if (!eq->bt_l0)
5490                 goto err_dma_alloc_l0;
5491
5492         if (mhop_num == 1) {
5493                 if (ba_num > (bt_chk_sz / BA_BYTE_LEN))
5494                         dev_err(dev, "ba_num %d is too large for 1 hop\n",
5495                                 ba_num);
5496
5497                 /* alloc buf */
5498                 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
5499                         if (eq_buf_cnt + 1 < ba_num) {
5500                                 size = buf_chk_sz;
5501                         } else {
5502                                 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
5503                                 size = (eq->entries - eqe_alloc) * eq->eqe_size;
5504                         }
5505                         eq->buf[i] = dma_alloc_coherent(dev, size,
5506                                                         &(eq->buf_dma[i]),
5507                                                         GFP_KERNEL);
5508                         if (!eq->buf[i])
5509                                 goto err_dma_alloc_buf;
5510
5511                         *(eq->bt_l0 + i) = eq->buf_dma[i];
5512
5513                         eq_buf_cnt++;
5514                         if (eq_buf_cnt >= ba_num)
5515                                 break;
5516                 }
5517                 eq->cur_eqe_ba = eq->buf_dma[0];
5518                 eq->nxt_eqe_ba = eq->buf_dma[1];
5519
5520         } else if (mhop_num == 2) {
5521                 /* alloc L1 BT and buf */
5522                 for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) {
5523                         eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
5524                                                           &(eq->l1_dma[i]),
5525                                                           GFP_KERNEL);
5526                         if (!eq->bt_l1[i])
5527                                 goto err_dma_alloc_l1;
5528                         *(eq->bt_l0 + i) = eq->l1_dma[i];
5529
5530                         for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5531                                 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5532                                 if (eq_buf_cnt + 1 < ba_num) {
5533                                         size = buf_chk_sz;
5534                                 } else {
5535                                         eqe_alloc = (buf_chk_sz / eq->eqe_size)
5536                                                     * idx;
5537                                         size = (eq->entries - eqe_alloc)
5538                                                 * eq->eqe_size;
5539                                 }
5540                                 eq->buf[idx] = dma_alloc_coherent(dev, size,
5541                                                                   &(eq->buf_dma[idx]),
5542                                                                   GFP_KERNEL);
5543                                 if (!eq->buf[idx])
5544                                         goto err_dma_alloc_buf;
5545
5546                                 *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
5547
5548                                 eq_buf_cnt++;
5549                                 if (eq_buf_cnt >= ba_num) {
5550                                         eq_alloc_done = 1;
5551                                         break;
5552                                 }
5553                         }
5554
5555                         if (eq_alloc_done)
5556                                 break;
5557                 }
5558                 eq->cur_eqe_ba = eq->buf_dma[0];
5559                 eq->nxt_eqe_ba = eq->buf_dma[1];
5560         }
5561
5562         eq->l0_last_num = i + 1;
5563         if (mhop_num == 2)
5564                 eq->l1_last_num = j + 1;
5565
5566         return 0;
5567
5568 err_dma_alloc_l1:
5569         dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5570         eq->bt_l0 = NULL;
5571         eq->l0_dma = 0;
5572         for (i -= 1; i >= 0; i--) {
5573                 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5574                                   eq->l1_dma[i]);
5575
5576                 for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5577                         idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5578                         dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5579                                           eq->buf_dma[idx]);
5580                 }
5581         }
5582         goto err_dma_alloc_l0;
5583
5584 err_dma_alloc_buf:
5585         dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5586         eq->bt_l0 = NULL;
5587         eq->l0_dma = 0;
5588
5589         if (mhop_num == 1)
5590                 for (i -= 1; i >= 0; i--)
5591                         dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5592                                           eq->buf_dma[i]);
5593         else if (mhop_num == 2) {
5594                 record_i = i;
5595                 record_j = j;
5596                 for (; i >= 0; i--) {
5597                         dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5598                                           eq->l1_dma[i]);
5599
5600                         for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) {
5601                                 if (i == record_i && j >= record_j)
5602                                         break;
5603
5604                                 idx = i * bt_chk_sz / BA_BYTE_LEN + j;
5605                                 dma_free_coherent(dev, buf_chk_sz,
5606                                                   eq->buf[idx],
5607                                                   eq->buf_dma[idx]);
5608                         }
5609                 }
5610         }
5611
5612 err_dma_alloc_l0:
5613         kfree(eq->bt_l1);
5614         eq->bt_l1 = NULL;
5615
5616 err_kcalloc_bt_l1:
5617         kfree(eq->l1_dma);
5618         eq->l1_dma = NULL;
5619
5620 err_kcalloc_l1_dma:
5621         kfree(eq->buf);
5622         eq->buf = NULL;
5623
5624 err_kcalloc_buf:
5625         kfree(eq->buf_dma);
5626         eq->buf_dma = NULL;
5627
5628         return -ENOMEM;
5629 }
5630
5631 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5632                                  struct hns_roce_eq *eq,
5633                                  unsigned int eq_cmd)
5634 {
5635         struct device *dev = hr_dev->dev;
5636         struct hns_roce_cmd_mailbox *mailbox;
5637         u32 buf_chk_sz = 0;
5638         int ret;
5639
5640         /* Allocate mailbox memory */
5641         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5642         if (IS_ERR(mailbox))
5643                 return PTR_ERR(mailbox);
5644
5645         if (!hr_dev->caps.eqe_hop_num) {
5646                 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5647
5648                 eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
5649                                        GFP_KERNEL);
5650                 if (!eq->buf_list) {
5651                         ret = -ENOMEM;
5652                         goto free_cmd_mbox;
5653                 }
5654
5655                 eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
5656                                                        &(eq->buf_list->map),
5657                                                        GFP_KERNEL);
5658                 if (!eq->buf_list->buf) {
5659                         ret = -ENOMEM;
5660                         goto err_alloc_buf;
5661                 }
5662
5663         } else {
5664                 ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
5665                 if (ret) {
5666                         ret = -ENOMEM;
5667                         goto free_cmd_mbox;
5668                 }
5669         }
5670
5671         hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
5672
5673         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5674                                 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5675         if (ret) {
5676                 dev_err(dev, "[mailbox cmd] create eqc failed.\n");
5677                 goto err_cmd_mbox;
5678         }
5679
5680         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5681
5682         return 0;
5683
5684 err_cmd_mbox:
5685         if (!hr_dev->caps.eqe_hop_num)
5686                 dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5687                                   eq->buf_list->map);
5688         else {
5689                 hns_roce_mhop_free_eq(hr_dev, eq);
5690                 goto free_cmd_mbox;
5691         }
5692
5693 err_alloc_buf:
5694         kfree(eq->buf_list);
5695
5696 free_cmd_mbox:
5697         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5698
5699         return ret;
5700 }
5701
5702 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
5703 {
5704         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5705         struct device *dev = hr_dev->dev;
5706         struct hns_roce_eq *eq;
5707         unsigned int eq_cmd;
5708         int irq_num;
5709         int eq_num;
5710         int other_num;
5711         int comp_num;
5712         int aeq_num;
5713         int i, j, k;
5714         int ret;
5715
5716         other_num = hr_dev->caps.num_other_vectors;
5717         comp_num = hr_dev->caps.num_comp_vectors;
5718         aeq_num = hr_dev->caps.num_aeq_vectors;
5719
5720         eq_num = comp_num + aeq_num;
5721         irq_num = eq_num + other_num;
5722
5723         eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
5724         if (!eq_table->eq)
5725                 return -ENOMEM;
5726
5727         for (i = 0; i < irq_num; i++) {
5728                 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5729                                                GFP_KERNEL);
5730                 if (!hr_dev->irq_names[i]) {
5731                         ret = -ENOMEM;
5732                         goto err_failed_kzalloc;
5733                 }
5734         }
5735
5736         /* create eq */
5737         for (j = 0; j < eq_num; j++) {
5738                 eq = &eq_table->eq[j];
5739                 eq->hr_dev = hr_dev;
5740                 eq->eqn = j;
5741                 if (j < comp_num) {
5742                         /* CEQ */
5743                         eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5744                         eq->type_flag = HNS_ROCE_CEQ;
5745                         eq->entries = hr_dev->caps.ceqe_depth;
5746                         eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5747                         eq->irq = hr_dev->irq[j + other_num + aeq_num];
5748                         eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5749                         eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5750                 } else {
5751                         /* AEQ */
5752                         eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5753                         eq->type_flag = HNS_ROCE_AEQ;
5754                         eq->entries = hr_dev->caps.aeqe_depth;
5755                         eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5756                         eq->irq = hr_dev->irq[j - comp_num + other_num];
5757                         eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5758                         eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5759                 }
5760
5761                 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
5762                 if (ret) {
5763                         dev_err(dev, "eq create failed.\n");
5764                         goto err_create_eq_fail;
5765                 }
5766         }
5767
5768         /* enable irq */
5769         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5770
5771         /* irq contains: abnormal + AEQ + CEQ*/
5772         for (k = 0; k < irq_num; k++)
5773                 if (k < other_num)
5774                         snprintf((char *)hr_dev->irq_names[k],
5775                                  HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);
5776                 else if (k < (other_num + aeq_num))
5777                         snprintf((char *)hr_dev->irq_names[k],
5778                                  HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5779                                  k - other_num);
5780                 else
5781                         snprintf((char *)hr_dev->irq_names[k],
5782                                  HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5783                                  k - other_num - aeq_num);
5784
5785         for (k = 0; k < irq_num; k++) {
5786                 if (k < other_num)
5787                         ret = request_irq(hr_dev->irq[k],
5788                                           hns_roce_v2_msix_interrupt_abn,
5789                                           0, hr_dev->irq_names[k], hr_dev);
5790
5791                 else if (k < (other_num + comp_num))
5792                         ret = request_irq(eq_table->eq[k - other_num].irq,
5793                                           hns_roce_v2_msix_interrupt_eq,
5794                                           0, hr_dev->irq_names[k + aeq_num],
5795                                           &eq_table->eq[k - other_num]);
5796                 else
5797                         ret = request_irq(eq_table->eq[k - other_num].irq,
5798                                           hns_roce_v2_msix_interrupt_eq,
5799                                           0, hr_dev->irq_names[k - comp_num],
5800                                           &eq_table->eq[k - other_num]);
5801                 if (ret) {
5802                         dev_err(dev, "Request irq error!\n");
5803                         goto err_request_irq_fail;
5804                 }
5805         }
5806
5807         hr_dev->irq_workq =
5808                 create_singlethread_workqueue("hns_roce_irq_workqueue");
5809         if (!hr_dev->irq_workq) {
5810                 dev_err(dev, "Create irq workqueue failed!\n");
5811                 ret = -ENOMEM;
5812                 goto err_request_irq_fail;
5813         }
5814
5815         return 0;
5816
5817 err_request_irq_fail:
5818         for (k -= 1; k >= 0; k--)
5819                 if (k < other_num)
5820                         free_irq(hr_dev->irq[k], hr_dev);
5821                 else
5822                         free_irq(eq_table->eq[k - other_num].irq,
5823                                  &eq_table->eq[k - other_num]);
5824
5825 err_create_eq_fail:
5826         for (j -= 1; j >= 0; j--)
5827                 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]);
5828
5829 err_failed_kzalloc:
5830         for (i -= 1; i >= 0; i--)
5831                 kfree(hr_dev->irq_names[i]);
5832         kfree(eq_table->eq);
5833
5834         return ret;
5835 }
5836
5837 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5838 {
5839         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5840         int irq_num;
5841         int eq_num;
5842         int i;
5843
5844         eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5845         irq_num = eq_num + hr_dev->caps.num_other_vectors;
5846
5847         /* Disable irq */
5848         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5849
5850         for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5851                 free_irq(hr_dev->irq[i], hr_dev);
5852
5853         for (i = 0; i < eq_num; i++) {
5854                 hns_roce_v2_destroy_eqc(hr_dev, i);
5855
5856                 free_irq(eq_table->eq[i].irq, &eq_table->eq[i]);
5857
5858                 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
5859         }
5860
5861         for (i = 0; i < irq_num; i++)
5862                 kfree(hr_dev->irq_names[i]);
5863
5864         kfree(eq_table->eq);
5865
5866         flush_workqueue(hr_dev->irq_workq);
5867         destroy_workqueue(hr_dev->irq_workq);
5868 }
5869
5870 static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
5871                                    struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
5872                                    u32 cqn, void *mb_buf, u64 *mtts_wqe,
5873                                    u64 *mtts_idx, dma_addr_t dma_handle_wqe,
5874                                    dma_addr_t dma_handle_idx)
5875 {
5876         struct hns_roce_srq_context *srq_context;
5877
5878         srq_context = mb_buf;
5879         memset(srq_context, 0, sizeof(*srq_context));
5880
5881         roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
5882                        SRQC_BYTE_4_SRQ_ST_S, 1);
5883
5884         roce_set_field(srq_context->byte_4_srqn_srqst,
5885                        SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
5886                        SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
5887                        (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
5888                        hr_dev->caps.srqwqe_hop_num));
5889         roce_set_field(srq_context->byte_4_srqn_srqst,
5890                        SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
5891                        ilog2(srq->max));
5892
5893         roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
5894                        SRQC_BYTE_4_SRQN_S, srq->srqn);
5895
5896         roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5897                        SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5898
5899         roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
5900                        SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
5901
5902         srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
5903
5904         roce_set_field(srq_context->byte_24_wqe_bt_ba,
5905                        SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
5906                        SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
5907                        cpu_to_le32(dma_handle_wqe >> 35));
5908
5909         roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
5910                        SRQC_BYTE_28_PD_S, pdn);
5911         roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
5912                        SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
5913                        fls(srq->max_gs - 1));
5914
5915         srq_context->idx_bt_ba = (u32)(dma_handle_idx >> 3);
5916         srq_context->idx_bt_ba = cpu_to_le32(srq_context->idx_bt_ba);
5917         roce_set_field(srq_context->rsv_idx_bt_ba,
5918                        SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
5919                        SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
5920                        cpu_to_le32(dma_handle_idx >> 35));
5921
5922         srq_context->idx_cur_blk_addr = (u32)(mtts_idx[0] >> PAGE_ADDR_SHIFT);
5923         srq_context->idx_cur_blk_addr =
5924                                      cpu_to_le32(srq_context->idx_cur_blk_addr);
5925         roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5926                        SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
5927                        SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
5928                        cpu_to_le32((mtts_idx[0]) >> (32 + PAGE_ADDR_SHIFT)));
5929         roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5930                        SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
5931                        SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
5932                        hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
5933                        hr_dev->caps.idx_hop_num);
5934
5935         roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5936                        SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
5937                        SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
5938                        hr_dev->caps.idx_ba_pg_sz);
5939         roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
5940                        SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
5941                        SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
5942                        hr_dev->caps.idx_buf_pg_sz);
5943
5944         srq_context->idx_nxt_blk_addr = (u32)(mtts_idx[1] >> PAGE_ADDR_SHIFT);
5945         srq_context->idx_nxt_blk_addr =
5946                                    cpu_to_le32(srq_context->idx_nxt_blk_addr);
5947         roce_set_field(srq_context->rsv_idxnxtblkaddr,
5948                        SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
5949                        SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
5950                        cpu_to_le32((mtts_idx[1]) >> (32 + PAGE_ADDR_SHIFT)));
5951         roce_set_field(srq_context->byte_56_xrc_cqn,
5952                        SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
5953                        cqn);
5954         roce_set_field(srq_context->byte_56_xrc_cqn,
5955                        SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
5956                        SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
5957                        hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
5958         roce_set_field(srq_context->byte_56_xrc_cqn,
5959                        SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
5960                        SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
5961                        hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
5962
5963         roce_set_bit(srq_context->db_record_addr_record_en,
5964                      SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
5965 }
5966
5967 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5968                                   struct ib_srq_attr *srq_attr,
5969                                   enum ib_srq_attr_mask srq_attr_mask,
5970                                   struct ib_udata *udata)
5971 {
5972         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5973         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5974         struct hns_roce_srq_context *srq_context;
5975         struct hns_roce_srq_context *srqc_mask;
5976         struct hns_roce_cmd_mailbox *mailbox;
5977         int ret;
5978
5979         if (srq_attr_mask & IB_SRQ_LIMIT) {
5980                 if (srq_attr->srq_limit >= srq->max)
5981                         return -EINVAL;
5982
5983                 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5984                 if (IS_ERR(mailbox))
5985                         return PTR_ERR(mailbox);
5986
5987                 srq_context = mailbox->buf;
5988                 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
5989
5990                 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
5991
5992                 roce_set_field(srq_context->byte_8_limit_wl,
5993                                SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5994                                SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
5995                 roce_set_field(srqc_mask->byte_8_limit_wl,
5996                                SRQC_BYTE_8_SRQ_LIMIT_WL_M,
5997                                SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
5998
5999                 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
6000                                         HNS_ROCE_CMD_MODIFY_SRQC,
6001                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
6002                 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6003                 if (ret) {
6004                         dev_err(hr_dev->dev,
6005                                 "MODIFY SRQ Failed to cmd mailbox.\n");
6006                         return ret;
6007                 }
6008         }
6009
6010         return 0;
6011 }
6012
6013 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
6014 {
6015         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6016         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6017         struct hns_roce_srq_context *srq_context;
6018         struct hns_roce_cmd_mailbox *mailbox;
6019         int limit_wl;
6020         int ret;
6021
6022         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6023         if (IS_ERR(mailbox))
6024                 return PTR_ERR(mailbox);
6025
6026         srq_context = mailbox->buf;
6027         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
6028                                 HNS_ROCE_CMD_QUERY_SRQC,
6029                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
6030         if (ret) {
6031                 dev_err(hr_dev->dev, "QUERY SRQ cmd process error\n");
6032                 goto out;
6033         }
6034
6035         limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
6036                                   SRQC_BYTE_8_SRQ_LIMIT_WL_M,
6037                                   SRQC_BYTE_8_SRQ_LIMIT_WL_S);
6038
6039         attr->srq_limit = limit_wl;
6040         attr->max_wr    = srq->max - 1;
6041         attr->max_sge   = srq->max_gs;
6042
6043         memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
6044
6045 out:
6046         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6047         return ret;
6048 }
6049
6050 static int find_empty_entry(struct hns_roce_idx_que *idx_que,
6051                             unsigned long size)
6052 {
6053         int wqe_idx;
6054
6055         if (unlikely(bitmap_full(idx_que->bitmap, size)))
6056                 return -ENOSPC;
6057
6058         wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
6059
6060         bitmap_set(idx_que->bitmap, wqe_idx, 1);
6061
6062         return wqe_idx;
6063 }
6064
6065 static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
6066                            int cur_idx, int wqe_idx)
6067 {
6068         unsigned int *addr;
6069
6070         addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
6071                                                    cur_idx * idx_que->entry_sz);
6072         *addr = wqe_idx;
6073 }
6074
6075 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
6076                                      const struct ib_recv_wr *wr,
6077                                      const struct ib_recv_wr **bad_wr)
6078 {
6079         struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
6080         struct hns_roce_srq *srq = to_hr_srq(ibsrq);
6081         struct hns_roce_v2_wqe_data_seg *dseg;
6082         struct hns_roce_v2_db srq_db;
6083         unsigned long flags;
6084         int ret = 0;
6085         int wqe_idx;
6086         void *wqe;
6087         int nreq;
6088         int ind;
6089         int i;
6090
6091         spin_lock_irqsave(&srq->lock, flags);
6092
6093         ind = srq->head & (srq->max - 1);
6094
6095         for (nreq = 0; wr; ++nreq, wr = wr->next) {
6096                 if (unlikely(wr->num_sge > srq->max_gs)) {
6097                         ret = -EINVAL;
6098                         *bad_wr = wr;
6099                         break;
6100                 }
6101
6102                 if (unlikely(srq->head == srq->tail)) {
6103                         ret = -ENOMEM;
6104                         *bad_wr = wr;
6105                         break;
6106                 }
6107
6108                 wqe_idx = find_empty_entry(&srq->idx_que, srq->max);
6109                 if (wqe_idx < 0) {
6110                         ret = -ENOMEM;
6111                         *bad_wr = wr;
6112                         break;
6113                 }
6114
6115                 fill_idx_queue(&srq->idx_que, ind, wqe_idx);
6116                 wqe = get_srq_wqe(srq, wqe_idx);
6117                 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
6118
6119                 for (i = 0; i < wr->num_sge; ++i) {
6120                         dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
6121                         dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
6122                         dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
6123                 }
6124
6125                 if (i < srq->max_gs) {
6126                         dseg[i].len = 0;
6127                         dseg[i].lkey = cpu_to_le32(0x100);
6128                         dseg[i].addr = 0;
6129                 }
6130
6131                 srq->wrid[wqe_idx] = wr->wr_id;
6132                 ind = (ind + 1) & (srq->max - 1);
6133         }
6134
6135         if (likely(nreq)) {
6136                 srq->head += nreq;
6137
6138                 /*
6139                  * Make sure that descriptors are written before
6140                  * doorbell record.
6141                  */
6142                 wmb();
6143
6144                 srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
6145                                 (srq->srqn & V2_DB_BYTE_4_TAG_M);
6146                 srq_db.parameter = srq->head;
6147
6148                 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
6149
6150         }
6151
6152         spin_unlock_irqrestore(&srq->lock, flags);
6153
6154         return ret;
6155 }
6156
6157 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
6158         .query_cqc_info = hns_roce_v2_query_cqc_info,
6159 };
6160
6161 static const struct ib_device_ops hns_roce_v2_dev_ops = {
6162         .destroy_qp = hns_roce_v2_destroy_qp,
6163         .modify_cq = hns_roce_v2_modify_cq,
6164         .poll_cq = hns_roce_v2_poll_cq,
6165         .post_recv = hns_roce_v2_post_recv,
6166         .post_send = hns_roce_v2_post_send,
6167         .query_qp = hns_roce_v2_query_qp,
6168         .req_notify_cq = hns_roce_v2_req_notify_cq,
6169 };
6170
6171 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6172         .modify_srq = hns_roce_v2_modify_srq,
6173         .post_srq_recv = hns_roce_v2_post_srq_recv,
6174         .query_srq = hns_roce_v2_query_srq,
6175 };
6176
6177 static const struct hns_roce_hw hns_roce_hw_v2 = {
6178         .cmq_init = hns_roce_v2_cmq_init,
6179         .cmq_exit = hns_roce_v2_cmq_exit,
6180         .hw_profile = hns_roce_v2_profile,
6181         .hw_init = hns_roce_v2_init,
6182         .hw_exit = hns_roce_v2_exit,
6183         .post_mbox = hns_roce_v2_post_mbox,
6184         .chk_mbox = hns_roce_v2_chk_mbox,
6185         .rst_prc_mbox = hns_roce_v2_rst_process_cmd,
6186         .set_gid = hns_roce_v2_set_gid,
6187         .set_mac = hns_roce_v2_set_mac,
6188         .write_mtpt = hns_roce_v2_write_mtpt,
6189         .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
6190         .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
6191         .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
6192         .write_cqc = hns_roce_v2_write_cqc,
6193         .set_hem = hns_roce_v2_set_hem,
6194         .clear_hem = hns_roce_v2_clear_hem,
6195         .modify_qp = hns_roce_v2_modify_qp,
6196         .query_qp = hns_roce_v2_query_qp,
6197         .destroy_qp = hns_roce_v2_destroy_qp,
6198         .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
6199         .modify_cq = hns_roce_v2_modify_cq,
6200         .post_send = hns_roce_v2_post_send,
6201         .post_recv = hns_roce_v2_post_recv,
6202         .req_notify_cq = hns_roce_v2_req_notify_cq,
6203         .poll_cq = hns_roce_v2_poll_cq,
6204         .init_eq = hns_roce_v2_init_eq_table,
6205         .cleanup_eq = hns_roce_v2_cleanup_eq_table,
6206         .write_srqc = hns_roce_v2_write_srqc,
6207         .modify_srq = hns_roce_v2_modify_srq,
6208         .query_srq = hns_roce_v2_query_srq,
6209         .post_srq_recv = hns_roce_v2_post_srq_recv,
6210         .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6211         .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
6212 };
6213
6214 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6215         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6216         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
6217         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6218         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
6219         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6220         /* required last entry */
6221         {0, }
6222 };
6223
6224 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6225
6226 static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6227                                   struct hnae3_handle *handle)
6228 {
6229         struct hns_roce_v2_priv *priv = hr_dev->priv;
6230         int i;
6231
6232         hr_dev->hw = &hns_roce_hw_v2;
6233         hr_dev->dfx = &hns_roce_dfx_hw_v2;
6234         hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6235         hr_dev->odb_offset = hr_dev->sdb_offset;
6236
6237         /* Get info from NIC driver. */
6238         hr_dev->reg_base = handle->rinfo.roce_io_base;
6239         hr_dev->caps.num_ports = 1;
6240         hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6241         hr_dev->iboe.phy_port[0] = 0;
6242
6243         addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6244                             hr_dev->iboe.netdevs[0]->dev_addr);
6245
6246         for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
6247                 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6248                                                 i + handle->rinfo.base_vector);
6249
6250         /* cmd issue mode: 0 is poll, 1 is event */
6251         hr_dev->cmd_mod = 1;
6252         hr_dev->loop_idc = 0;
6253
6254         hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6255         priv->handle = handle;
6256
6257         return 0;
6258 }
6259
6260 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6261 {
6262         struct hns_roce_dev *hr_dev;
6263         int ret;
6264
6265         hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
6266         if (!hr_dev)
6267                 return -ENOMEM;
6268
6269         hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6270         if (!hr_dev->priv) {
6271                 ret = -ENOMEM;
6272                 goto error_failed_kzalloc;
6273         }
6274
6275         hr_dev->pci_dev = handle->pdev;
6276         hr_dev->dev = &handle->pdev->dev;
6277
6278         ret = hns_roce_hw_v2_get_cfg(hr_dev, handle);
6279         if (ret) {
6280                 dev_err(hr_dev->dev, "Get Configuration failed!\n");
6281                 goto error_failed_get_cfg;
6282         }
6283
6284         ret = hns_roce_init(hr_dev);
6285         if (ret) {
6286                 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6287                 goto error_failed_get_cfg;
6288         }
6289
6290         handle->priv = hr_dev;
6291
6292         return 0;
6293
6294 error_failed_get_cfg:
6295         kfree(hr_dev->priv);
6296
6297 error_failed_kzalloc:
6298         ib_dealloc_device(&hr_dev->ib_dev);
6299
6300         return ret;
6301 }
6302
6303 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6304                                            bool reset)
6305 {
6306         struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
6307
6308         if (!hr_dev)
6309                 return;
6310
6311         handle->priv = NULL;
6312         hns_roce_exit(hr_dev);
6313         kfree(hr_dev->priv);
6314         ib_dealloc_device(&hr_dev->ib_dev);
6315 }
6316
6317 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6318 {
6319         const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
6320         const struct pci_device_id *id;
6321         struct device *dev = &handle->pdev->dev;
6322         int ret;
6323
6324         handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
6325
6326         if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
6327                 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6328                 goto reset_chk_err;
6329         }
6330
6331         id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
6332         if (!id)
6333                 return 0;
6334
6335         ret = __hns_roce_hw_v2_init_instance(handle);
6336         if (ret) {
6337                 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6338                 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
6339                 if (ops->ae_dev_resetting(handle) ||
6340                     ops->get_hw_reset_stat(handle))
6341                         goto reset_chk_err;
6342                 else
6343                         return ret;
6344         }
6345
6346         handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
6347
6348
6349         return 0;
6350
6351 reset_chk_err:
6352         dev_err(dev, "Device is busy in resetting state.\n"
6353                      "please retry later.\n");
6354
6355         return -EBUSY;
6356 }
6357
6358 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
6359                                            bool reset)
6360 {
6361         if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
6362                 return;
6363
6364         handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
6365
6366         __hns_roce_hw_v2_uninit_instance(handle, reset);
6367
6368         handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
6369 }
6370 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
6371 {
6372         struct hns_roce_dev *hr_dev;
6373         struct ib_event event;
6374
6375         if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
6376                 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6377                 return 0;
6378         }
6379
6380         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
6381         clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
6382
6383         hr_dev = (struct hns_roce_dev *)handle->priv;
6384         if (!hr_dev)
6385                 return 0;
6386
6387         hr_dev->is_reset = true;
6388         hr_dev->active = false;
6389         hr_dev->dis_db = true;
6390
6391         event.event = IB_EVENT_DEVICE_FATAL;
6392         event.device = &hr_dev->ib_dev;
6393         event.element.port_num = 1;
6394         ib_dispatch_event(&event);
6395
6396         return 0;
6397 }
6398
6399 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
6400 {
6401         struct device *dev = &handle->pdev->dev;
6402         int ret;
6403
6404         if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
6405                                &handle->rinfo.state)) {
6406                 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6407                 return 0;
6408         }
6409
6410         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
6411
6412         dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
6413         ret = __hns_roce_hw_v2_init_instance(handle);
6414         if (ret) {
6415                 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
6416                  * callback function, RoCE Engine reinitialize. If RoCE reinit
6417                  * failed, we should inform NIC driver.
6418                  */
6419                 handle->priv = NULL;
6420                 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
6421         } else {
6422                 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
6423                 dev_info(dev, "Reset done, RoCE client reinit finished.\n");
6424         }
6425
6426         return ret;
6427 }
6428
6429 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
6430 {
6431         if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
6432                 return 0;
6433
6434         handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
6435         dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
6436         msleep(100);
6437         __hns_roce_hw_v2_uninit_instance(handle, false);
6438
6439         return 0;
6440 }
6441
6442 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
6443                                        enum hnae3_reset_notify_type type)
6444 {
6445         int ret = 0;
6446
6447         switch (type) {
6448         case HNAE3_DOWN_CLIENT:
6449                 ret = hns_roce_hw_v2_reset_notify_down(handle);
6450                 break;
6451         case HNAE3_INIT_CLIENT:
6452                 ret = hns_roce_hw_v2_reset_notify_init(handle);
6453                 break;
6454         case HNAE3_UNINIT_CLIENT:
6455                 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
6456                 break;
6457         default:
6458                 break;
6459         }
6460
6461         return ret;
6462 }
6463
6464 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
6465         .init_instance = hns_roce_hw_v2_init_instance,
6466         .uninit_instance = hns_roce_hw_v2_uninit_instance,
6467         .reset_notify = hns_roce_hw_v2_reset_notify,
6468 };
6469
6470 static struct hnae3_client hns_roce_hw_v2_client = {
6471         .name = "hns_roce_hw_v2",
6472         .type = HNAE3_CLIENT_ROCE,
6473         .ops = &hns_roce_hw_v2_ops,
6474 };
6475
6476 static int __init hns_roce_hw_v2_init(void)
6477 {
6478         return hnae3_register_client(&hns_roce_hw_v2_client);
6479 }
6480
6481 static void __exit hns_roce_hw_v2_exit(void)
6482 {
6483         hnae3_unregister_client(&hns_roce_hw_v2_client);
6484 }
6485
6486 module_init(hns_roce_hw_v2_init);
6487 module_exit(hns_roce_hw_v2_exit);
6488
6489 MODULE_LICENSE("Dual BSD/GPL");
6490 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
6491 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
6492 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
6493 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");