net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
  10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20
  21 #define SMC_LLC_DATA_LEN                40
  22
  23 struct smc_llc_hdr {
  24         struct smc_wr_rx_hdr common;
  25         u8 length;      /* 44 */
  26 #if defined(__BIG_ENDIAN_BITFIELD)
  27         u8 reserved:4,
  28            add_link_rej_rsn:4;
  29 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  30         u8 add_link_rej_rsn:4,
  31            reserved:4;
  32 #endif
  33         u8 flags;
  34 };
  35
  36 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  37
  38 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  39         struct smc_llc_hdr hd;
  40         u8 sender_mac[ETH_ALEN];
  41         u8 sender_gid[SMC_GID_SIZE];
  42         u8 sender_qp_num[3];
  43         u8 link_num;
  44         u8 link_uid[SMC_LGR_ID_SIZE];
  45         u8 max_links;
  46         u8 reserved[9];
  47 };
  48
  49 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  50 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  51
  52 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  53
  54 struct smc_llc_msg_add_link {           /* type 0x02 */
  55         struct smc_llc_hdr hd;
  56         u8 sender_mac[ETH_ALEN];
  57         u8 reserved2[2];
  58         u8 sender_gid[SMC_GID_SIZE];
  59         u8 sender_qp_num[3];
  60         u8 link_num;
  61         u8 flags2;      /* QP mtu */
  62         u8 initial_psn[3];
  63         u8 reserved[8];
  64 };
  65
  66 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
  67 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
  68
  69 struct smc_llc_msg_del_link {           /* type 0x04 */
  70         struct smc_llc_hdr hd;
  71         u8 link_num;
  72         __be32 reason;
  73         u8 reserved[35];
  74 } __packed;                     /* format defined in RFC7609 */
  75
  76 struct smc_llc_msg_test_link {          /* type 0x07 */
  77         struct smc_llc_hdr hd;
  78         u8 user_data[16];
  79         u8 reserved[24];
  80 };
  81
  82 struct smc_rmb_rtoken {
  83         union {
  84                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
  85                                 /* is actually the num of rtokens, first */
  86                                 /* rtoken is always for the current link */
  87                 u8 link_id;     /* link id of the rtoken */
  88         };
  89         __be32 rmb_key;
  90         __be64 rmb_vaddr;
  91 } __packed;                     /* format defined in RFC7609 */
  92
  93 #define SMC_LLC_RKEYS_PER_MSG   3
  94
  95 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
  96         struct smc_llc_hdr hd;
  97         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
  98         u8 reserved;
  99 };
 100
 101 struct smc_llc_msg_confirm_rkey_cont {  /* type 0x08 */
 102         struct smc_llc_hdr hd;
 103         u8 num_rkeys;
 104         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
 105 };
 106
 107 #define SMC_LLC_DEL_RKEY_MAX    8
 108 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 109
 110 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 111         struct smc_llc_hdr hd;
 112         u8 num_rkeys;
 113         u8 err_mask;
 114         u8 reserved[2];
 115         __be32 rkey[8];
 116         u8 reserved2[4];
 117 };
 118
 119 union smc_llc_msg {
 120         struct smc_llc_msg_confirm_link confirm_link;
 121         struct smc_llc_msg_add_link add_link;
 122         struct smc_llc_msg_del_link delete_link;
 123
 124         struct smc_llc_msg_confirm_rkey confirm_rkey;
 125         struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
 126         struct smc_llc_msg_delete_rkey delete_rkey;
 127
 128         struct smc_llc_msg_test_link test_link;
 129         struct {
 130                 struct smc_llc_hdr hdr;
 131                 u8 data[SMC_LLC_DATA_LEN];
 132         } raw;
 133 };
 134
 135 #define SMC_LLC_FLAG_RESP               0x80
 136
 137 /********************************** send *************************************/
 138
 139 struct smc_llc_tx_pend {
 140 };
 141
 142 /* handler for send/transmission completion of an LLC msg */
 143 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 144                                struct smc_link *link,
 145                                enum ib_wc_status wc_status)
 146 {
 147         /* future work: handle wc_status error for recovery and failover */
 148 }
 149
 150 /**
 151  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 152  * @link: Pointer to SMC link used for sending LLC control message.
 153  * @wr_buf: Out variable returning pointer to work request payload buffer.
 154  * @pend: Out variable returning pointer to private pending WR tracking.
 155  *        It's the context the transmit complete handler will get.
 156  *
 157  * Reserves and pre-fills an entry for a pending work request send/tx.
 158  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 159  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 160  *
 161  * Return: 0 on success, otherwise an error value.
 162  */
 163 static int smc_llc_add_pending_send(struct smc_link *link,
 164                                     struct smc_wr_buf **wr_buf,
 165                                     struct smc_wr_tx_pend_priv **pend)
 166 {
 167         int rc;
 168
 169         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 170                                      pend);
 171         if (rc < 0)
 172                 return rc;
 173         BUILD_BUG_ON_MSG(
 174                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 175                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 176         BUILD_BUG_ON_MSG(
 177                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 178                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 179         BUILD_BUG_ON_MSG(
 180                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 181                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 182         return 0;
 183 }
 184
 185 /* high-level API to send LLC confirm link */
 186 int smc_llc_send_confirm_link(struct smc_link *link,
 187                               enum smc_llc_reqresp reqresp)
 188 {
 189         struct smc_link_group *lgr = smc_get_lgr(link);
 190         struct smc_llc_msg_confirm_link *confllc;
 191         struct smc_wr_tx_pend_priv *pend;
 192         struct smc_wr_buf *wr_buf;
 193         int rc;
 194
 195         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 196         if (rc)
 197                 return rc;
 198         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 199         memset(confllc, 0, sizeof(*confllc));
 200         confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
 201         confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
 202         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 203         if (reqresp == SMC_LLC_RESP)
 204                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 205         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 206                ETH_ALEN);
 207         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 208         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 209         confllc->link_num = link->link_id;
 210         memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
 211         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
 212         /* send llc message */
 213         rc = smc_wr_tx_send(link, pend);
 214         return rc;
 215 }
 216
 217 /* send LLC confirm rkey request */
 218 static int smc_llc_send_confirm_rkey(struct smc_link *link,
 219                                      struct smc_buf_desc *rmb_desc)
 220 {
 221         struct smc_llc_msg_confirm_rkey *rkeyllc;
 222         struct smc_wr_tx_pend_priv *pend;
 223         struct smc_wr_buf *wr_buf;
 224         int rc;
 225
 226         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 227         if (rc)
 228                 return rc;
 229         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 230         memset(rkeyllc, 0, sizeof(*rkeyllc));
 231         rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
 232         rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
 233         rkeyllc->rtoken[0].rmb_key =
 234                 htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
 235         rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
 236                 (u64)sg_dma_address(rmb_desc->sgt[SMC_SINGLE_LINK].sgl));
 237         /* send llc message */
 238         rc = smc_wr_tx_send(link, pend);
 239         return rc;
 240 }
 241
 242 /* send LLC delete rkey request */
 243 static int smc_llc_send_delete_rkey(struct smc_link *link,
 244                                     struct smc_buf_desc *rmb_desc)
 245 {
 246         struct smc_llc_msg_delete_rkey *rkeyllc;
 247         struct smc_wr_tx_pend_priv *pend;
 248         struct smc_wr_buf *wr_buf;
 249         int rc;
 250
 251         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 252         if (rc)
 253                 return rc;
 254         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 255         memset(rkeyllc, 0, sizeof(*rkeyllc));
 256         rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
 257         rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
 258         rkeyllc->num_rkeys = 1;
 259         rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey);
 260         /* send llc message */
 261         rc = smc_wr_tx_send(link, pend);
 262         return rc;
 263 }
 264
 265 /* prepare an add link message */
 266 static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
 267                                   struct smc_link *link, u8 mac[], u8 gid[],
 268                                   enum smc_llc_reqresp reqresp)
 269 {
 270         memset(addllc, 0, sizeof(*addllc));
 271         addllc->hd.common.type = SMC_LLC_ADD_LINK;
 272         addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 273         if (reqresp == SMC_LLC_RESP) {
 274                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 275                 /* always reject more links for now */
 276                 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 277                 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 278         }
 279         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 280         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 281 }
 282
 283 /* send ADD LINK request or response */
 284 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 285                           enum smc_llc_reqresp reqresp)
 286 {
 287         struct smc_llc_msg_add_link *addllc;
 288         struct smc_wr_tx_pend_priv *pend;
 289         struct smc_wr_buf *wr_buf;
 290         int rc;
 291
 292         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 293         if (rc)
 294                 return rc;
 295         addllc = (struct smc_llc_msg_add_link *)wr_buf;
 296         smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
 297         /* send llc message */
 298         rc = smc_wr_tx_send(link, pend);
 299         return rc;
 300 }
 301
 302 /* prepare a delete link message */
 303 static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
 304                                      struct smc_link *link,
 305                                      enum smc_llc_reqresp reqresp, bool orderly)
 306 {
 307         memset(delllc, 0, sizeof(*delllc));
 308         delllc->hd.common.type = SMC_LLC_DELETE_LINK;
 309         delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 310         if (reqresp == SMC_LLC_RESP)
 311                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 312         /* DEL_LINK_ALL because only 1 link supported */
 313         delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 314         if (orderly)
 315                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 316         delllc->link_num = link->link_id;
 317 }
 318
 319 /* send DELETE LINK request or response */
 320 int smc_llc_send_delete_link(struct smc_link *link,
 321                              enum smc_llc_reqresp reqresp, bool orderly)
 322 {
 323         struct smc_llc_msg_del_link *delllc;
 324         struct smc_wr_tx_pend_priv *pend;
 325         struct smc_wr_buf *wr_buf;
 326         int rc;
 327
 328         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 329         if (rc)
 330                 return rc;
 331         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 332         smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
 333         /* send llc message */
 334         rc = smc_wr_tx_send(link, pend);
 335         return rc;
 336 }
 337
 338 /* send LLC test link request */
 339 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 340 {
 341         struct smc_llc_msg_test_link *testllc;
 342         struct smc_wr_tx_pend_priv *pend;
 343         struct smc_wr_buf *wr_buf;
 344         int rc;
 345
 346         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 347         if (rc)
 348                 return rc;
 349         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 350         memset(testllc, 0, sizeof(*testllc));
 351         testllc->hd.common.type = SMC_LLC_TEST_LINK;
 352         testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
 353         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 354         /* send llc message */
 355         rc = smc_wr_tx_send(link, pend);
 356         return rc;
 357 }
 358
 359 struct smc_llc_send_work {
 360         struct work_struct work;
 361         struct smc_link *link;
 362         int llclen;
 363         union smc_llc_msg llcbuf;
 364 };
 365
 366 /* worker that sends a prepared message */
 367 static void smc_llc_send_message_work(struct work_struct *work)
 368 {
 369         struct smc_llc_send_work *llcwrk = container_of(work,
 370                                                 struct smc_llc_send_work, work);
 371         struct smc_wr_tx_pend_priv *pend;
 372         struct smc_wr_buf *wr_buf;
 373         int rc;
 374
 375         if (llcwrk->link->state == SMC_LNK_INACTIVE)
 376                 goto out;
 377         rc = smc_llc_add_pending_send(llcwrk->link, &wr_buf, &pend);
 378         if (rc)
 379                 goto out;
 380         memcpy(wr_buf, &llcwrk->llcbuf, llcwrk->llclen);
 381         smc_wr_tx_send(llcwrk->link, pend);
 382 out:
 383         kfree(llcwrk);
 384 }
 385
 386 /* copy llcbuf and schedule an llc send on link */
 387 static int smc_llc_send_message(struct smc_link *link, void *llcbuf, int llclen)
 388 {
 389         struct smc_llc_send_work *wrk = kmalloc(sizeof(*wrk), GFP_ATOMIC);
 390
 391         if (!wrk)
 392                 return -ENOMEM;
 393         INIT_WORK(&wrk->work, smc_llc_send_message_work);
 394         wrk->link = link;
 395         wrk->llclen = llclen;
 396         memcpy(&wrk->llcbuf, llcbuf, llclen);
 397         queue_work(link->llc_wq, &wrk->work);
 398         return 0;
 399 }
 400
 401 /********************************* receive ***********************************/
 402
 403 static void smc_llc_rx_confirm_link(struct smc_link *link,
 404                                     struct smc_llc_msg_confirm_link *llc)
 405 {
 406         struct smc_link_group *lgr = smc_get_lgr(link);
 407         int conf_rc;
 408
 409         /* RMBE eyecatchers are not supported */
 410         if (llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)
 411                 conf_rc = 0;
 412         else
 413                 conf_rc = ENOTSUPP;
 414
 415         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 416                 if (lgr->role == SMC_SERV &&
 417                     link->state == SMC_LNK_ACTIVATING) {
 418                         link->llc_confirm_resp_rc = conf_rc;
 419                         complete(&link->llc_confirm_resp);
 420                 }
 421         } else {
 422                 if (lgr->role == SMC_CLNT &&
 423                     link->state == SMC_LNK_ACTIVATING) {
 424                         link->llc_confirm_rc = conf_rc;
 425                         link->link_id = llc->link_num;
 426                         complete(&link->llc_confirm);
 427                 }
 428         }
 429 }
 430
 431 static void smc_llc_rx_add_link(struct smc_link *link,
 432                                 struct smc_llc_msg_add_link *llc)
 433 {
 434         struct smc_link_group *lgr = smc_get_lgr(link);
 435
 436         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 437                 if (link->state == SMC_LNK_ACTIVATING)
 438                         complete(&link->llc_add_resp);
 439         } else {
 440                 if (link->state == SMC_LNK_ACTIVATING) {
 441                         complete(&link->llc_add);
 442                         return;
 443                 }
 444
 445                 if (lgr->role == SMC_SERV) {
 446                         smc_llc_prep_add_link(llc, link,
 447                                         link->smcibdev->mac[link->ibport - 1],
 448                                         link->gid, SMC_LLC_REQ);
 449
 450                 } else {
 451                         smc_llc_prep_add_link(llc, link,
 452                                         link->smcibdev->mac[link->ibport - 1],
 453                                         link->gid, SMC_LLC_RESP);
 454                 }
 455                 smc_llc_send_message(link, llc, sizeof(*llc));
 456         }
 457 }
 458
 459 static void smc_llc_rx_delete_link(struct smc_link *link,
 460                                    struct smc_llc_msg_del_link *llc)
 461 {
 462         struct smc_link_group *lgr = smc_get_lgr(link);
 463
 464         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 465                 if (lgr->role == SMC_SERV)
 466                         smc_lgr_schedule_free_work_fast(lgr);
 467         } else {
 468                 smc_lgr_forget(lgr);
 469                 smc_llc_link_deleting(link);
 470                 if (lgr->role == SMC_SERV) {
 471                         /* client asks to delete this link, send request */
 472                         smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
 473                 } else {
 474                         /* server requests to delete this link, send response */
 475                         smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
 476                 }
 477                 smc_llc_send_message(link, llc, sizeof(*llc));
 478                 smc_lgr_schedule_free_work_fast(lgr);
 479         }
 480 }
 481
 482 static void smc_llc_rx_test_link(struct smc_link *link,
 483                                  struct smc_llc_msg_test_link *llc)
 484 {
 485         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 486                 if (link->state == SMC_LNK_ACTIVE)
 487                         complete(&link->llc_testlink_resp);
 488         } else {
 489                 llc->hd.flags |= SMC_LLC_FLAG_RESP;
 490                 smc_llc_send_message(link, llc, sizeof(*llc));
 491         }
 492 }
 493
 494 static void smc_llc_rx_confirm_rkey(struct smc_link *link,
 495                                     struct smc_llc_msg_confirm_rkey *llc)
 496 {
 497         int rc;
 498
 499         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 500                 link->llc_confirm_rkey_rc = llc->hd.flags &
 501                                             SMC_LLC_FLAG_RKEY_NEG;
 502                 complete(&link->llc_confirm_rkey);
 503         } else {
 504                 rc = smc_rtoken_add(smc_get_lgr(link),
 505                                     llc->rtoken[0].rmb_vaddr,
 506                                     llc->rtoken[0].rmb_key);
 507
 508                 /* ignore rtokens for other links, we have only one link */
 509
 510                 llc->hd.flags |= SMC_LLC_FLAG_RESP;
 511                 if (rc < 0)
 512                         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
 513                 smc_llc_send_message(link, llc, sizeof(*llc));
 514         }
 515 }
 516
 517 static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link,
 518                                       struct smc_llc_msg_confirm_rkey_cont *llc)
 519 {
 520         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 521                 /* unused as long as we don't send this type of msg */
 522         } else {
 523                 /* ignore rtokens for other links, we have only one link */
 524                 llc->hd.flags |= SMC_LLC_FLAG_RESP;
 525                 smc_llc_send_message(link, llc, sizeof(*llc));
 526         }
 527 }
 528
 529 static void smc_llc_rx_delete_rkey(struct smc_link *link,
 530                                    struct smc_llc_msg_delete_rkey *llc)
 531 {
 532         u8 err_mask = 0;
 533         int i, max;
 534
 535         if (llc->hd.flags & SMC_LLC_FLAG_RESP) {
 536                 link->llc_delete_rkey_rc = llc->hd.flags &
 537                                             SMC_LLC_FLAG_RKEY_NEG;
 538                 complete(&link->llc_delete_rkey);
 539         } else {
 540                 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
 541                 for (i = 0; i < max; i++) {
 542                         if (smc_rtoken_delete(smc_get_lgr(link), llc->rkey[i]))
 543                                 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
 544                 }
 545
 546                 if (err_mask) {
 547                         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
 548                         llc->err_mask = err_mask;
 549                 }
 550
 551                 llc->hd.flags |= SMC_LLC_FLAG_RESP;
 552                 smc_llc_send_message(link, llc, sizeof(*llc));
 553         }
 554 }
 555
 556 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
 557 {
 558         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
 559         union smc_llc_msg *llc = buf;
 560
 561         if (wc->byte_len < sizeof(*llc))
 562                 return; /* short message */
 563         if (llc->raw.hdr.length != sizeof(*llc))
 564                 return; /* invalid message */
 565         if (link->state == SMC_LNK_INACTIVE)
 566                 return; /* link not active, drop msg */
 567
 568         switch (llc->raw.hdr.common.type) {
 569         case SMC_LLC_TEST_LINK:
 570                 smc_llc_rx_test_link(link, &llc->test_link);
 571                 break;
 572         case SMC_LLC_CONFIRM_LINK:
 573                 smc_llc_rx_confirm_link(link, &llc->confirm_link);
 574                 break;
 575         case SMC_LLC_ADD_LINK:
 576                 smc_llc_rx_add_link(link, &llc->add_link);
 577                 break;
 578         case SMC_LLC_DELETE_LINK:
 579                 smc_llc_rx_delete_link(link, &llc->delete_link);
 580                 break;
 581         case SMC_LLC_CONFIRM_RKEY:
 582                 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);
 583                 break;
 584         case SMC_LLC_CONFIRM_RKEY_CONT:
 585                 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont);
 586                 break;
 587         case SMC_LLC_DELETE_RKEY:
 588                 smc_llc_rx_delete_rkey(link, &llc->delete_rkey);
 589                 break;
 590         }
 591 }
 592
 593 /***************************** worker, utils *********************************/
 594
 595 static void smc_llc_testlink_work(struct work_struct *work)
 596 {
 597         struct smc_link *link = container_of(to_delayed_work(work),
 598                                              struct smc_link, llc_testlink_wrk);
 599         unsigned long next_interval;
 600         unsigned long expire_time;
 601         u8 user_data[16] = { 0 };
 602         int rc;
 603
 604         if (link->state != SMC_LNK_ACTIVE)
 605                 return;         /* don't reschedule worker */
 606         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
 607         if (time_is_after_jiffies(expire_time)) {
 608                 next_interval = expire_time - jiffies;
 609                 goto out;
 610         }
 611         reinit_completion(&link->llc_testlink_resp);
 612         smc_llc_send_test_link(link, user_data);
 613         /* receive TEST LINK response over RoCE fabric */
 614         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
 615                                                        SMC_LLC_WAIT_TIME);
 616         if (rc <= 0) {
 617                 smc_lgr_terminate(smc_get_lgr(link));
 618                 return;
 619         }
 620         next_interval = link->llc_testlink_time;
 621 out:
 622         queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk,
 623                            next_interval);
 624 }
 625
 626 int smc_llc_link_init(struct smc_link *link)
 627 {
 628         struct smc_link_group *lgr = smc_get_lgr(link);
 629         link->llc_wq = alloc_ordered_workqueue("llc_wq-%x:%x)", WQ_MEM_RECLAIM,
 630                                                *((u32 *)lgr->id),
 631                                                link->link_id);
 632         if (!link->llc_wq)
 633                 return -ENOMEM;
 634         init_completion(&link->llc_confirm);
 635         init_completion(&link->llc_confirm_resp);
 636         init_completion(&link->llc_add);
 637         init_completion(&link->llc_add_resp);
 638         init_completion(&link->llc_confirm_rkey);
 639         init_completion(&link->llc_delete_rkey);
 640         mutex_init(&link->llc_delete_rkey_mutex);
 641         init_completion(&link->llc_testlink_resp);
 642         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
 643         return 0;
 644 }
 645
 646 void smc_llc_link_active(struct smc_link *link, int testlink_time)
 647 {
 648         link->state = SMC_LNK_ACTIVE;
 649         if (testlink_time) {
 650                 link->llc_testlink_time = testlink_time * HZ;
 651                 queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk,
 652                                    link->llc_testlink_time);
 653         }
 654 }
 655
 656 void smc_llc_link_deleting(struct smc_link *link)
 657 {
 658         link->state = SMC_LNK_DELETING;
 659 }
 660
 661 /* called in tasklet context */
 662 void smc_llc_link_inactive(struct smc_link *link)
 663 {
 664         link->state = SMC_LNK_INACTIVE;
 665         cancel_delayed_work(&link->llc_testlink_wrk);
 666 }
 667
 668 /* called in worker context */
 669 void smc_llc_link_clear(struct smc_link *link)
 670 {
 671         flush_workqueue(link->llc_wq);
 672         destroy_workqueue(link->llc_wq);
 673 }
 674
 675 /* register a new rtoken at the remote peer */
 676 int smc_llc_do_confirm_rkey(struct smc_link *link,
 677                             struct smc_buf_desc *rmb_desc)
 678 {
 679         int rc;
 680
 681         /* protected by mutex smc_create_lgr_pending */
 682         reinit_completion(&link->llc_confirm_rkey);
 683         rc = smc_llc_send_confirm_rkey(link, rmb_desc);
 684         if (rc)
 685                 return rc;
 686         /* receive CONFIRM RKEY response from server over RoCE fabric */
 687         rc = wait_for_completion_interruptible_timeout(&link->llc_confirm_rkey,
 688                                                        SMC_LLC_WAIT_TIME);
 689         if (rc <= 0 || link->llc_confirm_rkey_rc)
 690                 return -EFAULT;
 691         return 0;
 692 }
 693
 694 /* unregister an rtoken at the remote peer */
 695 int smc_llc_do_delete_rkey(struct smc_link *link,
 696                            struct smc_buf_desc *rmb_desc)
 697 {
 698         int rc;
 699
 700         mutex_lock(&link->llc_delete_rkey_mutex);
 701         reinit_completion(&link->llc_delete_rkey);
 702         rc = smc_llc_send_delete_rkey(link, rmb_desc);
 703         if (rc)
 704                 goto out;
 705         /* receive DELETE RKEY response from server over RoCE fabric */
 706         rc = wait_for_completion_interruptible_timeout(&link->llc_delete_rkey,
 707                                                        SMC_LLC_WAIT_TIME);
 708         if (rc <= 0 || link->llc_delete_rkey_rc)
 709                 rc = -EFAULT;
 710         else
 711                 rc = 0;
 712 out:
 713         mutex_unlock(&link->llc_delete_rkey_mutex);
 714         return rc;
 715 }
 716
 717 /***************************** init, exit, misc ******************************/
 718
 719 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
 720         {
 721                 .handler        = smc_llc_rx_handler,
 722                 .type           = SMC_LLC_CONFIRM_LINK
 723         },
 724         {
 725                 .handler        = smc_llc_rx_handler,
 726                 .type           = SMC_LLC_TEST_LINK
 727         },
 728         {
 729                 .handler        = smc_llc_rx_handler,
 730                 .type           = SMC_LLC_ADD_LINK
 731         },
 732         {
 733                 .handler        = smc_llc_rx_handler,
 734                 .type           = SMC_LLC_DELETE_LINK
 735         },
 736         {
 737                 .handler        = smc_llc_rx_handler,
 738                 .type           = SMC_LLC_CONFIRM_RKEY
 739         },
 740         {
 741                 .handler        = smc_llc_rx_handler,
 742                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
 743         },
 744         {
 745                 .handler        = smc_llc_rx_handler,
 746                 .type           = SMC_LLC_DELETE_RKEY
 747         },
 748         {
 749                 .handler        = NULL,
 750         }
 751 };
 752
 753 int __init smc_llc_init(void)
 754 {
 755         struct smc_wr_rx_handler *handler;
 756         int rc = 0;
 757
 758         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
 759                 INIT_HLIST_NODE(&handler->list);
 760                 rc = smc_wr_rx_register_handler(handler);
 761                 if (rc)
 762                         break;
 763         }
 764         return rc;
 765 }