drivers/target/iscsi/iscsi_target_util.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*******************************************************************************
   3  * This file contains the iSCSI Target specific utility functions.
   4  *
   5  * (c) Copyright 2007-2013 Datera, Inc.
   6  *
   7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   8  *
   9  ******************************************************************************/
  10
  11 #include <linux/list.h>
  12 #include <linux/sched/signal.h>
  13 #include <net/ipv6.h>         /* ipv6_addr_equal() */
  14 #include <scsi/scsi_tcq.h>
  15 #include <scsi/iscsi_proto.h>
  16 #include <target/target_core_base.h>
  17 #include <target/target_core_fabric.h>
  18 #include <target/iscsi/iscsi_transport.h>
  19
  20 #include <target/iscsi/iscsi_target_core.h>
  21 #include "iscsi_target_parameters.h"
  22 #include "iscsi_target_seq_pdu_list.h"
  23 #include "iscsi_target_datain_values.h"
  24 #include "iscsi_target_erl0.h"
  25 #include "iscsi_target_erl1.h"
  26 #include "iscsi_target_erl2.h"
  27 #include "iscsi_target_tpg.h"
  28 #include "iscsi_target_util.h"
  29 #include "iscsi_target.h"
  30
  31 #define PRINT_BUFF(buff, len)                                   \
  32 {                                                               \
  33         int zzz;                                                \
  34                                                                 \
  35         pr_debug("%d:\n", __LINE__);                            \
  36         for (zzz = 0; zzz < len; zzz++) {                       \
  37                 if (zzz % 16 == 0) {                            \
  38                         if (zzz)                                \
  39                                 pr_debug("\n");                 \
  40                         pr_debug("%4i: ", zzz);                 \
  41                 }                                               \
  42                 pr_debug("%02x ", (unsigned char) (buff)[zzz]); \
  43         }                                                       \
  44         if ((len + 1) % 16)                                     \
  45                 pr_debug("\n");                                 \
  46 }
  47
  48 extern struct list_head g_tiqn_list;
  49 extern spinlock_t tiqn_lock;
  50
  51 int iscsit_add_r2t_to_list(
  52         struct iscsi_cmd *cmd,
  53         u32 offset,
  54         u32 xfer_len,
  55         int recovery,
  56         u32 r2t_sn)
  57 {
  58         struct iscsi_r2t *r2t;
  59
  60         lockdep_assert_held(&cmd->r2t_lock);
  61
  62         WARN_ON_ONCE((s32)xfer_len < 0);
  63
  64         r2t = kmem_cache_zalloc(lio_r2t_cache, GFP_ATOMIC);
  65         if (!r2t) {
  66                 pr_err("Unable to allocate memory for struct iscsi_r2t.\n");
  67                 return -1;
  68         }
  69         INIT_LIST_HEAD(&r2t->r2t_list);
  70
  71         r2t->recovery_r2t = recovery;
  72         r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;
  73         r2t->offset = offset;
  74         r2t->xfer_len = xfer_len;
  75         list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);
  76         spin_unlock_bh(&cmd->r2t_lock);
  77
  78         iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);
  79
  80         spin_lock_bh(&cmd->r2t_lock);
  81         return 0;
  82 }
  83
  84 struct iscsi_r2t *iscsit_get_r2t_for_eos(
  85         struct iscsi_cmd *cmd,
  86         u32 offset,
  87         u32 length)
  88 {
  89         struct iscsi_r2t *r2t;
  90
  91         spin_lock_bh(&cmd->r2t_lock);
  92         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
  93                 if ((r2t->offset <= offset) &&
  94                     (r2t->offset + r2t->xfer_len) >= (offset + length)) {
  95                         spin_unlock_bh(&cmd->r2t_lock);
  96                         return r2t;
  97                 }
  98         }
  99         spin_unlock_bh(&cmd->r2t_lock);
 100
 101         pr_err("Unable to locate R2T for Offset: %u, Length:"
 102                         " %u\n", offset, length);
 103         return NULL;
 104 }
 105
 106 struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)
 107 {
 108         struct iscsi_r2t *r2t;
 109
 110         spin_lock_bh(&cmd->r2t_lock);
 111         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 112                 if (!r2t->sent_r2t) {
 113                         spin_unlock_bh(&cmd->r2t_lock);
 114                         return r2t;
 115                 }
 116         }
 117         spin_unlock_bh(&cmd->r2t_lock);
 118
 119         pr_err("Unable to locate next R2T to send for ITT:"
 120                         " 0x%08x.\n", cmd->init_task_tag);
 121         return NULL;
 122 }
 123
 124 void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)
 125 {
 126         lockdep_assert_held(&cmd->r2t_lock);
 127
 128         list_del(&r2t->r2t_list);
 129         kmem_cache_free(lio_r2t_cache, r2t);
 130 }
 131
 132 void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)
 133 {
 134         struct iscsi_r2t *r2t, *r2t_tmp;
 135
 136         spin_lock_bh(&cmd->r2t_lock);
 137         list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)
 138                 iscsit_free_r2t(r2t, cmd);
 139         spin_unlock_bh(&cmd->r2t_lock);
 140 }
 141
 142 static int iscsit_wait_for_tag(struct se_session *se_sess, int state, int *cpup)
 143 {
 144         int tag = -1;
 145         DEFINE_SBQ_WAIT(wait);
 146         struct sbq_wait_state *ws;
 147         struct sbitmap_queue *sbq;
 148
 149         if (state == TASK_RUNNING)
 150                 return tag;
 151
 152         sbq = &se_sess->sess_tag_pool;
 153         ws = &sbq->ws[0];
 154         for (;;) {
 155                 sbitmap_prepare_to_wait(sbq, ws, &wait, state);
 156                 if (signal_pending_state(state, current))
 157                         break;
 158                 tag = sbitmap_queue_get(sbq, cpup);
 159                 if (tag >= 0)
 160                         break;
 161                 schedule();
 162         }
 163
 164         sbitmap_finish_wait(sbq, ws, &wait);
 165         return tag;
 166 }
 167
 168 /*
 169  * May be called from software interrupt (timer) context for allocating
 170  * iSCSI NopINs.
 171  */
 172 struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, int state)
 173 {
 174         struct iscsi_cmd *cmd;
 175         struct se_session *se_sess = conn->sess->se_sess;
 176         int size, tag, cpu;
 177
 178         tag = sbitmap_queue_get(&se_sess->sess_tag_pool, &cpu);
 179         if (tag < 0)
 180                 tag = iscsit_wait_for_tag(se_sess, state, &cpu);
 181         if (tag < 0)
 182                 return NULL;
 183
 184         size = sizeof(struct iscsi_cmd) + conn->conn_transport->priv_size;
 185         cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));
 186         memset(cmd, 0, size);
 187
 188         cmd->se_cmd.map_tag = tag;
 189         cmd->se_cmd.map_cpu = cpu;
 190         cmd->conn = conn;
 191         cmd->data_direction = DMA_NONE;
 192         INIT_LIST_HEAD(&cmd->i_conn_node);
 193         INIT_LIST_HEAD(&cmd->datain_list);
 194         INIT_LIST_HEAD(&cmd->cmd_r2t_list);
 195         spin_lock_init(&cmd->datain_lock);
 196         spin_lock_init(&cmd->dataout_timeout_lock);
 197         spin_lock_init(&cmd->istate_lock);
 198         spin_lock_init(&cmd->error_lock);
 199         spin_lock_init(&cmd->r2t_lock);
 200         timer_setup(&cmd->dataout_timer, iscsit_handle_dataout_timeout, 0);
 201
 202         return cmd;
 203 }
 204 EXPORT_SYMBOL(iscsit_allocate_cmd);
 205
 206 struct iscsi_seq *iscsit_get_seq_holder_for_datain(
 207         struct iscsi_cmd *cmd,
 208         u32 seq_send_order)
 209 {
 210         u32 i;
 211
 212         for (i = 0; i < cmd->seq_count; i++)
 213                 if (cmd->seq_list[i].seq_send_order == seq_send_order)
 214                         return &cmd->seq_list[i];
 215
 216         return NULL;
 217 }
 218
 219 struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)
 220 {
 221         u32 i;
 222
 223         if (!cmd->seq_list) {
 224                 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
 225                 return NULL;
 226         }
 227
 228         for (i = 0; i < cmd->seq_count; i++) {
 229                 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
 230                         continue;
 231                 if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {
 232                         cmd->seq_send_order++;
 233                         return &cmd->seq_list[i];
 234                 }
 235         }
 236
 237         return NULL;
 238 }
 239
 240 struct iscsi_r2t *iscsit_get_holder_for_r2tsn(
 241         struct iscsi_cmd *cmd,
 242         u32 r2t_sn)
 243 {
 244         struct iscsi_r2t *r2t;
 245
 246         spin_lock_bh(&cmd->r2t_lock);
 247         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 248                 if (r2t->r2t_sn == r2t_sn) {
 249                         spin_unlock_bh(&cmd->r2t_lock);
 250                         return r2t;
 251                 }
 252         }
 253         spin_unlock_bh(&cmd->r2t_lock);
 254
 255         return NULL;
 256 }
 257
 258 static inline int iscsit_check_received_cmdsn(struct iscsi_session *sess, u32 cmdsn)
 259 {
 260         u32 max_cmdsn;
 261         int ret;
 262
 263         /*
 264          * This is the proper method of checking received CmdSN against
 265          * ExpCmdSN and MaxCmdSN values, as well as accounting for out
 266          * or order CmdSNs due to multiple connection sessions and/or
 267          * CRC failures.
 268          */
 269         max_cmdsn = atomic_read(&sess->max_cmd_sn);
 270         if (iscsi_sna_gt(cmdsn, max_cmdsn)) {
 271                 pr_err("Received CmdSN: 0x%08x is greater than"
 272                        " MaxCmdSN: 0x%08x, ignoring.\n", cmdsn, max_cmdsn);
 273                 ret = CMDSN_MAXCMDSN_OVERRUN;
 274
 275         } else if (cmdsn == sess->exp_cmd_sn) {
 276                 sess->exp_cmd_sn++;
 277                 pr_debug("Received CmdSN matches ExpCmdSN,"
 278                       " incremented ExpCmdSN to: 0x%08x\n",
 279                       sess->exp_cmd_sn);
 280                 ret = CMDSN_NORMAL_OPERATION;
 281
 282         } else if (iscsi_sna_gt(cmdsn, sess->exp_cmd_sn)) {
 283                 pr_debug("Received CmdSN: 0x%08x is greater"
 284                       " than ExpCmdSN: 0x%08x, not acknowledging.\n",
 285                       cmdsn, sess->exp_cmd_sn);
 286                 ret = CMDSN_HIGHER_THAN_EXP;
 287
 288         } else {
 289                 pr_err("Received CmdSN: 0x%08x is less than"
 290                        " ExpCmdSN: 0x%08x, ignoring.\n", cmdsn,
 291                        sess->exp_cmd_sn);
 292                 ret = CMDSN_LOWER_THAN_EXP;
 293         }
 294
 295         return ret;
 296 }
 297
 298 /*
 299  * Commands may be received out of order if MC/S is in use.
 300  * Ensure they are executed in CmdSN order.
 301  */
 302 int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 303                         unsigned char *buf, __be32 cmdsn)
 304 {
 305         int ret, cmdsn_ret;
 306         bool reject = false;
 307         u8 reason = ISCSI_REASON_BOOKMARK_NO_RESOURCES;
 308
 309         mutex_lock(&conn->sess->cmdsn_mutex);
 310
 311         cmdsn_ret = iscsit_check_received_cmdsn(conn->sess, be32_to_cpu(cmdsn));
 312         switch (cmdsn_ret) {
 313         case CMDSN_NORMAL_OPERATION:
 314                 ret = iscsit_execute_cmd(cmd, 0);
 315                 if ((ret >= 0) && !list_empty(&conn->sess->sess_ooo_cmdsn_list))
 316                         iscsit_execute_ooo_cmdsns(conn->sess);
 317                 else if (ret < 0) {
 318                         reject = true;
 319                         ret = CMDSN_ERROR_CANNOT_RECOVER;
 320                 }
 321                 break;
 322         case CMDSN_HIGHER_THAN_EXP:
 323                 ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));
 324                 if (ret < 0) {
 325                         reject = true;
 326                         ret = CMDSN_ERROR_CANNOT_RECOVER;
 327                         break;
 328                 }
 329                 ret = CMDSN_HIGHER_THAN_EXP;
 330                 break;
 331         case CMDSN_LOWER_THAN_EXP:
 332         case CMDSN_MAXCMDSN_OVERRUN:
 333         default:
 334                 cmd->i_state = ISTATE_REMOVE;
 335                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 336                 /*
 337                  * Existing callers for iscsit_sequence_cmd() will silently
 338                  * ignore commands with CMDSN_LOWER_THAN_EXP, so force this
 339                  * return for CMDSN_MAXCMDSN_OVERRUN as well..
 340                  */
 341                 ret = CMDSN_LOWER_THAN_EXP;
 342                 break;
 343         }
 344         mutex_unlock(&conn->sess->cmdsn_mutex);
 345
 346         if (reject)
 347                 iscsit_reject_cmd(cmd, reason, buf);
 348
 349         return ret;
 350 }
 351 EXPORT_SYMBOL(iscsit_sequence_cmd);
 352
 353 int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)
 354 {
 355         struct iscsi_conn *conn = cmd->conn;
 356         struct se_cmd *se_cmd = &cmd->se_cmd;
 357         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 358         u32 payload_length = ntoh24(hdr->dlength);
 359
 360         if (conn->sess->sess_ops->InitialR2T) {
 361                 pr_err("Received unexpected unsolicited data"
 362                         " while InitialR2T=Yes, protocol error.\n");
 363                 transport_send_check_condition_and_sense(se_cmd,
 364                                 TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
 365                 return -1;
 366         }
 367
 368         if ((cmd->first_burst_len + payload_length) >
 369              conn->sess->sess_ops->FirstBurstLength) {
 370                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 371                         " for this Unsolicited DataOut Burst.\n",
 372                         (cmd->first_burst_len + payload_length),
 373                                 conn->sess->sess_ops->FirstBurstLength);
 374                 transport_send_check_condition_and_sense(se_cmd,
 375                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 376                 return -1;
 377         }
 378
 379         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))
 380                 return 0;
 381
 382         if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&
 383             ((cmd->first_burst_len + payload_length) !=
 384               conn->sess->sess_ops->FirstBurstLength)) {
 385                 pr_err("Unsolicited non-immediate data received %u"
 386                         " does not equal FirstBurstLength: %u, and does"
 387                         " not equal ExpXferLen %u.\n",
 388                         (cmd->first_burst_len + payload_length),
 389                         conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);
 390                 transport_send_check_condition_and_sense(se_cmd,
 391                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 392                 return -1;
 393         }
 394         return 0;
 395 }
 396
 397 struct iscsi_cmd *iscsit_find_cmd_from_itt(
 398         struct iscsi_conn *conn,
 399         itt_t init_task_tag)
 400 {
 401         struct iscsi_cmd *cmd;
 402
 403         spin_lock_bh(&conn->cmd_lock);
 404         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 405                 if (cmd->init_task_tag == init_task_tag) {
 406                         spin_unlock_bh(&conn->cmd_lock);
 407                         return cmd;
 408                 }
 409         }
 410         spin_unlock_bh(&conn->cmd_lock);
 411
 412         pr_err("Unable to locate ITT: 0x%08x on CID: %hu",
 413                         init_task_tag, conn->cid);
 414         return NULL;
 415 }
 416 EXPORT_SYMBOL(iscsit_find_cmd_from_itt);
 417
 418 struct iscsi_cmd *iscsit_find_cmd_from_itt_or_dump(
 419         struct iscsi_conn *conn,
 420         itt_t init_task_tag,
 421         u32 length)
 422 {
 423         struct iscsi_cmd *cmd;
 424
 425         spin_lock_bh(&conn->cmd_lock);
 426         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 427                 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)
 428                         continue;
 429                 if (cmd->init_task_tag == init_task_tag) {
 430                         spin_unlock_bh(&conn->cmd_lock);
 431                         return cmd;
 432                 }
 433         }
 434         spin_unlock_bh(&conn->cmd_lock);
 435
 436         pr_err("Unable to locate ITT: 0x%08x on CID: %hu,"
 437                         " dumping payload\n", init_task_tag, conn->cid);
 438         if (length)
 439                 iscsit_dump_data_payload(conn, length, 1);
 440
 441         return NULL;
 442 }
 443 EXPORT_SYMBOL(iscsit_find_cmd_from_itt_or_dump);
 444
 445 struct iscsi_cmd *iscsit_find_cmd_from_ttt(
 446         struct iscsi_conn *conn,
 447         u32 targ_xfer_tag)
 448 {
 449         struct iscsi_cmd *cmd = NULL;
 450
 451         spin_lock_bh(&conn->cmd_lock);
 452         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 453                 if (cmd->targ_xfer_tag == targ_xfer_tag) {
 454                         spin_unlock_bh(&conn->cmd_lock);
 455                         return cmd;
 456                 }
 457         }
 458         spin_unlock_bh(&conn->cmd_lock);
 459
 460         pr_err("Unable to locate TTT: 0x%08x on CID: %hu\n",
 461                         targ_xfer_tag, conn->cid);
 462         return NULL;
 463 }
 464
 465 int iscsit_find_cmd_for_recovery(
 466         struct iscsi_session *sess,
 467         struct iscsi_cmd **cmd_ptr,
 468         struct iscsi_conn_recovery **cr_ptr,
 469         itt_t init_task_tag)
 470 {
 471         struct iscsi_cmd *cmd = NULL;
 472         struct iscsi_conn_recovery *cr;
 473         /*
 474          * Scan through the inactive connection recovery list's command list.
 475          * If init_task_tag matches the command is still alligent.
 476          */
 477         spin_lock(&sess->cr_i_lock);
 478         list_for_each_entry(cr, &sess->cr_inactive_list, cr_list) {
 479                 spin_lock(&cr->conn_recovery_cmd_lock);
 480                 list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
 481                         if (cmd->init_task_tag == init_task_tag) {
 482                                 spin_unlock(&cr->conn_recovery_cmd_lock);
 483                                 spin_unlock(&sess->cr_i_lock);
 484
 485                                 *cr_ptr = cr;
 486                                 *cmd_ptr = cmd;
 487                                 return -2;
 488                         }
 489                 }
 490                 spin_unlock(&cr->conn_recovery_cmd_lock);
 491         }
 492         spin_unlock(&sess->cr_i_lock);
 493         /*
 494          * Scan through the active connection recovery list's command list.
 495          * If init_task_tag matches the command is ready to be reassigned.
 496          */
 497         spin_lock(&sess->cr_a_lock);
 498         list_for_each_entry(cr, &sess->cr_active_list, cr_list) {
 499                 spin_lock(&cr->conn_recovery_cmd_lock);
 500                 list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
 501                         if (cmd->init_task_tag == init_task_tag) {
 502                                 spin_unlock(&cr->conn_recovery_cmd_lock);
 503                                 spin_unlock(&sess->cr_a_lock);
 504
 505                                 *cr_ptr = cr;
 506                                 *cmd_ptr = cmd;
 507                                 return 0;
 508                         }
 509                 }
 510                 spin_unlock(&cr->conn_recovery_cmd_lock);
 511         }
 512         spin_unlock(&sess->cr_a_lock);
 513
 514         return -1;
 515 }
 516
 517 void iscsit_add_cmd_to_immediate_queue(
 518         struct iscsi_cmd *cmd,
 519         struct iscsi_conn *conn,
 520         u8 state)
 521 {
 522         struct iscsi_queue_req *qr;
 523
 524         qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
 525         if (!qr) {
 526                 pr_err("Unable to allocate memory for"
 527                                 " struct iscsi_queue_req\n");
 528                 return;
 529         }
 530         INIT_LIST_HEAD(&qr->qr_list);
 531         qr->cmd = cmd;
 532         qr->state = state;
 533
 534         spin_lock_bh(&conn->immed_queue_lock);
 535         list_add_tail(&qr->qr_list, &conn->immed_queue_list);
 536         atomic_inc(&cmd->immed_queue_count);
 537         atomic_set(&conn->check_immediate_queue, 1);
 538         spin_unlock_bh(&conn->immed_queue_lock);
 539
 540         wake_up(&conn->queues_wq);
 541 }
 542 EXPORT_SYMBOL(iscsit_add_cmd_to_immediate_queue);
 543
 544 struct iscsi_queue_req *iscsit_get_cmd_from_immediate_queue(struct iscsi_conn *conn)
 545 {
 546         struct iscsi_queue_req *qr;
 547
 548         spin_lock_bh(&conn->immed_queue_lock);
 549         if (list_empty(&conn->immed_queue_list)) {
 550                 spin_unlock_bh(&conn->immed_queue_lock);
 551                 return NULL;
 552         }
 553         qr = list_first_entry(&conn->immed_queue_list,
 554                               struct iscsi_queue_req, qr_list);
 555
 556         list_del(&qr->qr_list);
 557         if (qr->cmd)
 558                 atomic_dec(&qr->cmd->immed_queue_count);
 559         spin_unlock_bh(&conn->immed_queue_lock);
 560
 561         return qr;
 562 }
 563
 564 static void iscsit_remove_cmd_from_immediate_queue(
 565         struct iscsi_cmd *cmd,
 566         struct iscsi_conn *conn)
 567 {
 568         struct iscsi_queue_req *qr, *qr_tmp;
 569
 570         spin_lock_bh(&conn->immed_queue_lock);
 571         if (!atomic_read(&cmd->immed_queue_count)) {
 572                 spin_unlock_bh(&conn->immed_queue_lock);
 573                 return;
 574         }
 575
 576         list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
 577                 if (qr->cmd != cmd)
 578                         continue;
 579
 580                 atomic_dec(&qr->cmd->immed_queue_count);
 581                 list_del(&qr->qr_list);
 582                 kmem_cache_free(lio_qr_cache, qr);
 583         }
 584         spin_unlock_bh(&conn->immed_queue_lock);
 585
 586         if (atomic_read(&cmd->immed_queue_count)) {
 587                 pr_err("ITT: 0x%08x immed_queue_count: %d\n",
 588                         cmd->init_task_tag,
 589                         atomic_read(&cmd->immed_queue_count));
 590         }
 591 }
 592
 593 int iscsit_add_cmd_to_response_queue(
 594         struct iscsi_cmd *cmd,
 595         struct iscsi_conn *conn,
 596         u8 state)
 597 {
 598         struct iscsi_queue_req *qr;
 599
 600         qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
 601         if (!qr) {
 602                 pr_err("Unable to allocate memory for"
 603                         " struct iscsi_queue_req\n");
 604                 return -ENOMEM;
 605         }
 606         INIT_LIST_HEAD(&qr->qr_list);
 607         qr->cmd = cmd;
 608         qr->state = state;
 609
 610         spin_lock_bh(&conn->response_queue_lock);
 611         list_add_tail(&qr->qr_list, &conn->response_queue_list);
 612         atomic_inc(&cmd->response_queue_count);
 613         spin_unlock_bh(&conn->response_queue_lock);
 614
 615         wake_up(&conn->queues_wq);
 616         return 0;
 617 }
 618
 619 struct iscsi_queue_req *iscsit_get_cmd_from_response_queue(struct iscsi_conn *conn)
 620 {
 621         struct iscsi_queue_req *qr;
 622
 623         spin_lock_bh(&conn->response_queue_lock);
 624         if (list_empty(&conn->response_queue_list)) {
 625                 spin_unlock_bh(&conn->response_queue_lock);
 626                 return NULL;
 627         }
 628
 629         qr = list_first_entry(&conn->response_queue_list,
 630                               struct iscsi_queue_req, qr_list);
 631
 632         list_del(&qr->qr_list);
 633         if (qr->cmd)
 634                 atomic_dec(&qr->cmd->response_queue_count);
 635         spin_unlock_bh(&conn->response_queue_lock);
 636
 637         return qr;
 638 }
 639
 640 static void iscsit_remove_cmd_from_response_queue(
 641         struct iscsi_cmd *cmd,
 642         struct iscsi_conn *conn)
 643 {
 644         struct iscsi_queue_req *qr, *qr_tmp;
 645
 646         spin_lock_bh(&conn->response_queue_lock);
 647         if (!atomic_read(&cmd->response_queue_count)) {
 648                 spin_unlock_bh(&conn->response_queue_lock);
 649                 return;
 650         }
 651
 652         list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
 653                                 qr_list) {
 654                 if (qr->cmd != cmd)
 655                         continue;
 656
 657                 atomic_dec(&qr->cmd->response_queue_count);
 658                 list_del(&qr->qr_list);
 659                 kmem_cache_free(lio_qr_cache, qr);
 660         }
 661         spin_unlock_bh(&conn->response_queue_lock);
 662
 663         if (atomic_read(&cmd->response_queue_count)) {
 664                 pr_err("ITT: 0x%08x response_queue_count: %d\n",
 665                         cmd->init_task_tag,
 666                         atomic_read(&cmd->response_queue_count));
 667         }
 668 }
 669
 670 bool iscsit_conn_all_queues_empty(struct iscsi_conn *conn)
 671 {
 672         bool empty;
 673
 674         spin_lock_bh(&conn->immed_queue_lock);
 675         empty = list_empty(&conn->immed_queue_list);
 676         spin_unlock_bh(&conn->immed_queue_lock);
 677
 678         if (!empty)
 679                 return empty;
 680
 681         spin_lock_bh(&conn->response_queue_lock);
 682         empty = list_empty(&conn->response_queue_list);
 683         spin_unlock_bh(&conn->response_queue_lock);
 684
 685         return empty;
 686 }
 687
 688 void iscsit_free_queue_reqs_for_conn(struct iscsi_conn *conn)
 689 {
 690         struct iscsi_queue_req *qr, *qr_tmp;
 691
 692         spin_lock_bh(&conn->immed_queue_lock);
 693         list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
 694                 list_del(&qr->qr_list);
 695                 if (qr->cmd)
 696                         atomic_dec(&qr->cmd->immed_queue_count);
 697
 698                 kmem_cache_free(lio_qr_cache, qr);
 699         }
 700         spin_unlock_bh(&conn->immed_queue_lock);
 701
 702         spin_lock_bh(&conn->response_queue_lock);
 703         list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
 704                         qr_list) {
 705                 list_del(&qr->qr_list);
 706                 if (qr->cmd)
 707                         atomic_dec(&qr->cmd->response_queue_count);
 708
 709                 kmem_cache_free(lio_qr_cache, qr);
 710         }
 711         spin_unlock_bh(&conn->response_queue_lock);
 712 }
 713
 714 void iscsit_release_cmd(struct iscsi_cmd *cmd)
 715 {
 716         struct iscsi_session *sess;
 717         struct se_cmd *se_cmd = &cmd->se_cmd;
 718
 719         WARN_ON(!list_empty(&cmd->i_conn_node));
 720
 721         if (cmd->conn)
 722                 sess = cmd->conn->sess;
 723         else
 724                 sess = cmd->sess;
 725
 726         BUG_ON(!sess || !sess->se_sess);
 727
 728         kfree(cmd->buf_ptr);
 729         kfree(cmd->pdu_list);
 730         kfree(cmd->seq_list);
 731         kfree(cmd->tmr_req);
 732         kfree(cmd->overflow_buf);
 733         kfree(cmd->iov_data);
 734         kfree(cmd->text_in_ptr);
 735
 736         target_free_tag(sess->se_sess, se_cmd);
 737 }
 738 EXPORT_SYMBOL(iscsit_release_cmd);
 739
 740 void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool check_queues)
 741 {
 742         struct iscsi_conn *conn = cmd->conn;
 743
 744         WARN_ON(!list_empty(&cmd->i_conn_node));
 745
 746         if (cmd->data_direction == DMA_TO_DEVICE) {
 747                 iscsit_stop_dataout_timer(cmd);
 748                 iscsit_free_r2ts_from_list(cmd);
 749         }
 750         if (cmd->data_direction == DMA_FROM_DEVICE)
 751                 iscsit_free_all_datain_reqs(cmd);
 752
 753         if (conn && check_queues) {
 754                 iscsit_remove_cmd_from_immediate_queue(cmd, conn);
 755                 iscsit_remove_cmd_from_response_queue(cmd, conn);
 756         }
 757
 758         if (conn && conn->conn_transport->iscsit_unmap_cmd)
 759                 conn->conn_transport->iscsit_unmap_cmd(conn, cmd);
 760 }
 761
 762 void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)
 763 {
 764         struct se_cmd *se_cmd = cmd->se_cmd.se_tfo ? &cmd->se_cmd : NULL;
 765         int rc;
 766
 767         WARN_ON(!list_empty(&cmd->i_conn_node));
 768
 769         __iscsit_free_cmd(cmd, shutdown);
 770         if (se_cmd) {
 771                 rc = transport_generic_free_cmd(se_cmd, shutdown);
 772                 if (!rc && shutdown && se_cmd->se_sess) {
 773                         __iscsit_free_cmd(cmd, shutdown);
 774                         target_put_sess_cmd(se_cmd);
 775                 }
 776         } else {
 777                 iscsit_release_cmd(cmd);
 778         }
 779 }
 780 EXPORT_SYMBOL(iscsit_free_cmd);
 781
 782 int iscsit_check_session_usage_count(struct iscsi_session *sess)
 783 {
 784         spin_lock_bh(&sess->session_usage_lock);
 785         if (sess->session_usage_count != 0) {
 786                 sess->session_waiting_on_uc = 1;
 787                 spin_unlock_bh(&sess->session_usage_lock);
 788                 if (in_interrupt())
 789                         return 2;
 790
 791                 wait_for_completion(&sess->session_waiting_on_uc_comp);
 792                 return 1;
 793         }
 794         spin_unlock_bh(&sess->session_usage_lock);
 795
 796         return 0;
 797 }
 798
 799 void iscsit_dec_session_usage_count(struct iscsi_session *sess)
 800 {
 801         spin_lock_bh(&sess->session_usage_lock);
 802         sess->session_usage_count--;
 803
 804         if (!sess->session_usage_count && sess->session_waiting_on_uc)
 805                 complete(&sess->session_waiting_on_uc_comp);
 806
 807         spin_unlock_bh(&sess->session_usage_lock);
 808 }
 809
 810 void iscsit_inc_session_usage_count(struct iscsi_session *sess)
 811 {
 812         spin_lock_bh(&sess->session_usage_lock);
 813         sess->session_usage_count++;
 814         spin_unlock_bh(&sess->session_usage_lock);
 815 }
 816
 817 struct iscsi_conn *iscsit_get_conn_from_cid(struct iscsi_session *sess, u16 cid)
 818 {
 819         struct iscsi_conn *conn;
 820
 821         spin_lock_bh(&sess->conn_lock);
 822         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
 823                 if ((conn->cid == cid) &&
 824                     (conn->conn_state == TARG_CONN_STATE_LOGGED_IN)) {
 825                         iscsit_inc_conn_usage_count(conn);
 826                         spin_unlock_bh(&sess->conn_lock);
 827                         return conn;
 828                 }
 829         }
 830         spin_unlock_bh(&sess->conn_lock);
 831
 832         return NULL;
 833 }
 834
 835 struct iscsi_conn *iscsit_get_conn_from_cid_rcfr(struct iscsi_session *sess, u16 cid)
 836 {
 837         struct iscsi_conn *conn;
 838
 839         spin_lock_bh(&sess->conn_lock);
 840         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
 841                 if (conn->cid == cid) {
 842                         iscsit_inc_conn_usage_count(conn);
 843                         spin_lock(&conn->state_lock);
 844                         atomic_set(&conn->connection_wait_rcfr, 1);
 845                         spin_unlock(&conn->state_lock);
 846                         spin_unlock_bh(&sess->conn_lock);
 847                         return conn;
 848                 }
 849         }
 850         spin_unlock_bh(&sess->conn_lock);
 851
 852         return NULL;
 853 }
 854
 855 void iscsit_check_conn_usage_count(struct iscsi_conn *conn)
 856 {
 857         spin_lock_bh(&conn->conn_usage_lock);
 858         if (conn->conn_usage_count != 0) {
 859                 conn->conn_waiting_on_uc = 1;
 860                 spin_unlock_bh(&conn->conn_usage_lock);
 861
 862                 wait_for_completion(&conn->conn_waiting_on_uc_comp);
 863                 return;
 864         }
 865         spin_unlock_bh(&conn->conn_usage_lock);
 866 }
 867
 868 void iscsit_dec_conn_usage_count(struct iscsi_conn *conn)
 869 {
 870         spin_lock_bh(&conn->conn_usage_lock);
 871         conn->conn_usage_count--;
 872
 873         if (!conn->conn_usage_count && conn->conn_waiting_on_uc)
 874                 complete(&conn->conn_waiting_on_uc_comp);
 875
 876         spin_unlock_bh(&conn->conn_usage_lock);
 877 }
 878
 879 void iscsit_inc_conn_usage_count(struct iscsi_conn *conn)
 880 {
 881         spin_lock_bh(&conn->conn_usage_lock);
 882         conn->conn_usage_count++;
 883         spin_unlock_bh(&conn->conn_usage_lock);
 884 }
 885
 886 static int iscsit_add_nopin(struct iscsi_conn *conn, int want_response)
 887 {
 888         u8 state;
 889         struct iscsi_cmd *cmd;
 890
 891         cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);
 892         if (!cmd)
 893                 return -1;
 894
 895         cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;
 896         state = (want_response) ? ISTATE_SEND_NOPIN_WANT_RESPONSE :
 897                                 ISTATE_SEND_NOPIN_NO_RESPONSE;
 898         cmd->init_task_tag = RESERVED_ITT;
 899         cmd->targ_xfer_tag = (want_response) ?
 900                              session_get_next_ttt(conn->sess) : 0xFFFFFFFF;
 901         spin_lock_bh(&conn->cmd_lock);
 902         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 903         spin_unlock_bh(&conn->cmd_lock);
 904
 905         if (want_response)
 906                 iscsit_start_nopin_response_timer(conn);
 907         iscsit_add_cmd_to_immediate_queue(cmd, conn, state);
 908
 909         return 0;
 910 }
 911
 912 void iscsit_handle_nopin_response_timeout(struct timer_list *t)
 913 {
 914         struct iscsi_conn *conn = from_timer(conn, t, nopin_response_timer);
 915         struct iscsi_session *sess = conn->sess;
 916
 917         iscsit_inc_conn_usage_count(conn);
 918
 919         spin_lock_bh(&conn->nopin_timer_lock);
 920         if (conn->nopin_response_timer_flags & ISCSI_TF_STOP) {
 921                 spin_unlock_bh(&conn->nopin_timer_lock);
 922                 iscsit_dec_conn_usage_count(conn);
 923                 return;
 924         }
 925
 926         pr_err("Did not receive response to NOPIN on CID: %hu, failing"
 927                 " connection for I_T Nexus %s,i,0x%6phN,%s,t,0x%02x\n",
 928                 conn->cid, sess->sess_ops->InitiatorName, sess->isid,
 929                 sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt);
 930         conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
 931         spin_unlock_bh(&conn->nopin_timer_lock);
 932
 933         iscsit_fill_cxn_timeout_err_stats(sess);
 934         iscsit_cause_connection_reinstatement(conn, 0);
 935         iscsit_dec_conn_usage_count(conn);
 936 }
 937
 938 void iscsit_mod_nopin_response_timer(struct iscsi_conn *conn)
 939 {
 940         struct iscsi_session *sess = conn->sess;
 941         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
 942
 943         spin_lock_bh(&conn->nopin_timer_lock);
 944         if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
 945                 spin_unlock_bh(&conn->nopin_timer_lock);
 946                 return;
 947         }
 948
 949         mod_timer(&conn->nopin_response_timer,
 950                 (get_jiffies_64() + na->nopin_response_timeout * HZ));
 951         spin_unlock_bh(&conn->nopin_timer_lock);
 952 }
 953
 954 void iscsit_start_nopin_response_timer(struct iscsi_conn *conn)
 955 {
 956         struct iscsi_session *sess = conn->sess;
 957         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
 958
 959         spin_lock_bh(&conn->nopin_timer_lock);
 960         if (conn->nopin_response_timer_flags & ISCSI_TF_RUNNING) {
 961                 spin_unlock_bh(&conn->nopin_timer_lock);
 962                 return;
 963         }
 964
 965         conn->nopin_response_timer_flags &= ~ISCSI_TF_STOP;
 966         conn->nopin_response_timer_flags |= ISCSI_TF_RUNNING;
 967         mod_timer(&conn->nopin_response_timer,
 968                   jiffies + na->nopin_response_timeout * HZ);
 969
 970         pr_debug("Started NOPIN Response Timer on CID: %d to %u"
 971                 " seconds\n", conn->cid, na->nopin_response_timeout);
 972         spin_unlock_bh(&conn->nopin_timer_lock);
 973 }
 974
 975 void iscsit_stop_nopin_response_timer(struct iscsi_conn *conn)
 976 {
 977         spin_lock_bh(&conn->nopin_timer_lock);
 978         if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
 979                 spin_unlock_bh(&conn->nopin_timer_lock);
 980                 return;
 981         }
 982         conn->nopin_response_timer_flags |= ISCSI_TF_STOP;
 983         spin_unlock_bh(&conn->nopin_timer_lock);
 984
 985         del_timer_sync(&conn->nopin_response_timer);
 986
 987         spin_lock_bh(&conn->nopin_timer_lock);
 988         conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
 989         spin_unlock_bh(&conn->nopin_timer_lock);
 990 }
 991
 992 void iscsit_handle_nopin_timeout(struct timer_list *t)
 993 {
 994         struct iscsi_conn *conn = from_timer(conn, t, nopin_timer);
 995
 996         iscsit_inc_conn_usage_count(conn);
 997
 998         spin_lock_bh(&conn->nopin_timer_lock);
 999         if (conn->nopin_timer_flags & ISCSI_TF_STOP) {
1000                 spin_unlock_bh(&conn->nopin_timer_lock);
1001                 iscsit_dec_conn_usage_count(conn);
1002                 return;
1003         }
1004         conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1005         spin_unlock_bh(&conn->nopin_timer_lock);
1006
1007         iscsit_add_nopin(conn, 1);
1008         iscsit_dec_conn_usage_count(conn);
1009 }
1010
1011 void __iscsit_start_nopin_timer(struct iscsi_conn *conn)
1012 {
1013         struct iscsi_session *sess = conn->sess;
1014         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1015
1016         lockdep_assert_held(&conn->nopin_timer_lock);
1017
1018         /*
1019         * NOPIN timeout is disabled.
1020          */
1021         if (!na->nopin_timeout)
1022                 return;
1023
1024         if (conn->nopin_timer_flags & ISCSI_TF_RUNNING)
1025                 return;
1026
1027         conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1028         conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1029         mod_timer(&conn->nopin_timer, jiffies + na->nopin_timeout * HZ);
1030
1031         pr_debug("Started NOPIN Timer on CID: %d at %u second"
1032                 " interval\n", conn->cid, na->nopin_timeout);
1033 }
1034
1035 void iscsit_start_nopin_timer(struct iscsi_conn *conn)
1036 {
1037         spin_lock_bh(&conn->nopin_timer_lock);
1038         __iscsit_start_nopin_timer(conn);
1039         spin_unlock_bh(&conn->nopin_timer_lock);
1040 }
1041
1042 void iscsit_stop_nopin_timer(struct iscsi_conn *conn)
1043 {
1044         spin_lock_bh(&conn->nopin_timer_lock);
1045         if (!(conn->nopin_timer_flags & ISCSI_TF_RUNNING)) {
1046                 spin_unlock_bh(&conn->nopin_timer_lock);
1047                 return;
1048         }
1049         conn->nopin_timer_flags |= ISCSI_TF_STOP;
1050         spin_unlock_bh(&conn->nopin_timer_lock);
1051
1052         del_timer_sync(&conn->nopin_timer);
1053
1054         spin_lock_bh(&conn->nopin_timer_lock);
1055         conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1056         spin_unlock_bh(&conn->nopin_timer_lock);
1057 }
1058
1059 int iscsit_send_tx_data(
1060         struct iscsi_cmd *cmd,
1061         struct iscsi_conn *conn,
1062         int use_misc)
1063 {
1064         int tx_sent, tx_size;
1065         u32 iov_count;
1066         struct kvec *iov;
1067
1068 send_data:
1069         tx_size = cmd->tx_size;
1070
1071         if (!use_misc) {
1072                 iov = &cmd->iov_data[0];
1073                 iov_count = cmd->iov_data_count;
1074         } else {
1075                 iov = &cmd->iov_misc[0];
1076                 iov_count = cmd->iov_misc_count;
1077         }
1078
1079         tx_sent = tx_data(conn, &iov[0], iov_count, tx_size);
1080         if (tx_size != tx_sent) {
1081                 if (tx_sent == -EAGAIN) {
1082                         pr_err("tx_data() returned -EAGAIN\n");
1083                         goto send_data;
1084                 } else
1085                         return -1;
1086         }
1087         cmd->tx_size = 0;
1088
1089         return 0;
1090 }
1091
1092 int iscsit_fe_sendpage_sg(
1093         struct iscsi_cmd *cmd,
1094         struct iscsi_conn *conn)
1095 {
1096         struct scatterlist *sg = cmd->first_data_sg;
1097         struct kvec iov;
1098         u32 tx_hdr_size, data_len;
1099         u32 offset = cmd->first_data_sg_off;
1100         int tx_sent, iov_off;
1101
1102 send_hdr:
1103         tx_hdr_size = ISCSI_HDR_LEN;
1104         if (conn->conn_ops->HeaderDigest)
1105                 tx_hdr_size += ISCSI_CRC_LEN;
1106
1107         iov.iov_base = cmd->pdu;
1108         iov.iov_len = tx_hdr_size;
1109
1110         tx_sent = tx_data(conn, &iov, 1, tx_hdr_size);
1111         if (tx_hdr_size != tx_sent) {
1112                 if (tx_sent == -EAGAIN) {
1113                         pr_err("tx_data() returned -EAGAIN\n");
1114                         goto send_hdr;
1115                 }
1116                 return -1;
1117         }
1118
1119         data_len = cmd->tx_size - tx_hdr_size - cmd->padding;
1120         /*
1121          * Set iov_off used by padding and data digest tx_data() calls below
1122          * in order to determine proper offset into cmd->iov_data[]
1123          */
1124         if (conn->conn_ops->DataDigest) {
1125                 data_len -= ISCSI_CRC_LEN;
1126                 if (cmd->padding)
1127                         iov_off = (cmd->iov_data_count - 2);
1128                 else
1129                         iov_off = (cmd->iov_data_count - 1);
1130         } else {
1131                 iov_off = (cmd->iov_data_count - 1);
1132         }
1133         /*
1134          * Perform sendpage() for each page in the scatterlist
1135          */
1136         while (data_len) {
1137                 u32 space = (sg->length - offset);
1138                 u32 sub_len = min_t(u32, data_len, space);
1139 send_pg:
1140                 tx_sent = conn->sock->ops->sendpage(conn->sock,
1141                                         sg_page(sg), sg->offset + offset, sub_len, 0);
1142                 if (tx_sent != sub_len) {
1143                         if (tx_sent == -EAGAIN) {
1144                                 pr_err("tcp_sendpage() returned"
1145                                                 " -EAGAIN\n");
1146                                 goto send_pg;
1147                         }
1148
1149                         pr_err("tcp_sendpage() failure: %d\n",
1150                                         tx_sent);
1151                         return -1;
1152                 }
1153
1154                 data_len -= sub_len;
1155                 offset = 0;
1156                 sg = sg_next(sg);
1157         }
1158
1159 send_padding:
1160         if (cmd->padding) {
1161                 struct kvec *iov_p = &cmd->iov_data[iov_off++];
1162
1163                 tx_sent = tx_data(conn, iov_p, 1, cmd->padding);
1164                 if (cmd->padding != tx_sent) {
1165                         if (tx_sent == -EAGAIN) {
1166                                 pr_err("tx_data() returned -EAGAIN\n");
1167                                 goto send_padding;
1168                         }
1169                         return -1;
1170                 }
1171         }
1172
1173 send_datacrc:
1174         if (conn->conn_ops->DataDigest) {
1175                 struct kvec *iov_d = &cmd->iov_data[iov_off];
1176
1177                 tx_sent = tx_data(conn, iov_d, 1, ISCSI_CRC_LEN);
1178                 if (ISCSI_CRC_LEN != tx_sent) {
1179                         if (tx_sent == -EAGAIN) {
1180                                 pr_err("tx_data() returned -EAGAIN\n");
1181                                 goto send_datacrc;
1182                         }
1183                         return -1;
1184                 }
1185         }
1186
1187         return 0;
1188 }
1189
1190 /*
1191  *      This function is used for mainly sending a ISCSI_TARG_LOGIN_RSP PDU
1192  *      back to the Initiator when an expection condition occurs with the
1193  *      errors set in status_class and status_detail.
1194  *
1195  *      Parameters:     iSCSI Connection, Status Class, Status Detail.
1196  *      Returns:        0 on success, -1 on error.
1197  */
1198 int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail)
1199 {
1200         struct iscsi_login_rsp *hdr;
1201         struct iscsi_login *login = conn->conn_login;
1202
1203         login->login_failed = 1;
1204         iscsit_collect_login_stats(conn, status_class, status_detail);
1205
1206         memset(&login->rsp[0], 0, ISCSI_HDR_LEN);
1207
1208         hdr     = (struct iscsi_login_rsp *)&login->rsp[0];
1209         hdr->opcode             = ISCSI_OP_LOGIN_RSP;
1210         hdr->status_class       = status_class;
1211         hdr->status_detail      = status_detail;
1212         hdr->itt                = conn->login_itt;
1213
1214         return conn->conn_transport->iscsit_put_login_tx(conn, login, 0);
1215 }
1216
1217 void iscsit_print_session_params(struct iscsi_session *sess)
1218 {
1219         struct iscsi_conn *conn;
1220
1221         pr_debug("-----------------------------[Session Params for"
1222                 " SID: %u]-----------------------------\n", sess->sid);
1223         spin_lock_bh(&sess->conn_lock);
1224         list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
1225                 iscsi_dump_conn_ops(conn->conn_ops);
1226         spin_unlock_bh(&sess->conn_lock);
1227
1228         iscsi_dump_sess_ops(sess->sess_ops);
1229 }
1230
1231 static int iscsit_do_rx_data(
1232         struct iscsi_conn *conn,
1233         struct iscsi_data_count *count)
1234 {
1235         int data = count->data_length, rx_loop = 0, total_rx = 0;
1236         struct msghdr msg;
1237
1238         if (!conn || !conn->sock || !conn->conn_ops)
1239                 return -1;
1240
1241         memset(&msg, 0, sizeof(struct msghdr));
1242         iov_iter_kvec(&msg.msg_iter, READ, count->iov, count->iov_count, data);
1243
1244         while (msg_data_left(&msg)) {
1245                 rx_loop = sock_recvmsg(conn->sock, &msg, MSG_WAITALL);
1246                 if (rx_loop <= 0) {
1247                         pr_debug("rx_loop: %d total_rx: %d\n",
1248                                 rx_loop, total_rx);
1249                         return rx_loop;
1250                 }
1251                 total_rx += rx_loop;
1252                 pr_debug("rx_loop: %d, total_rx: %d, data: %d\n",
1253                                 rx_loop, total_rx, data);
1254         }
1255
1256         return total_rx;
1257 }
1258
1259 int rx_data(
1260         struct iscsi_conn *conn,
1261         struct kvec *iov,
1262         int iov_count,
1263         int data)
1264 {
1265         struct iscsi_data_count c;
1266
1267         if (!conn || !conn->sock || !conn->conn_ops)
1268                 return -1;
1269
1270         memset(&c, 0, sizeof(struct iscsi_data_count));
1271         c.iov = iov;
1272         c.iov_count = iov_count;
1273         c.data_length = data;
1274         c.type = ISCSI_RX_DATA;
1275
1276         return iscsit_do_rx_data(conn, &c);
1277 }
1278
1279 int tx_data(
1280         struct iscsi_conn *conn,
1281         struct kvec *iov,
1282         int iov_count,
1283         int data)
1284 {
1285         struct msghdr msg;
1286         int total_tx = 0;
1287
1288         if (!conn || !conn->sock || !conn->conn_ops)
1289                 return -1;
1290
1291         if (data <= 0) {
1292                 pr_err("Data length is: %d\n", data);
1293                 return -1;
1294         }
1295
1296         memset(&msg, 0, sizeof(struct msghdr));
1297
1298         iov_iter_kvec(&msg.msg_iter, WRITE, iov, iov_count, data);
1299
1300         while (msg_data_left(&msg)) {
1301                 int tx_loop = sock_sendmsg(conn->sock, &msg);
1302                 if (tx_loop <= 0) {
1303                         pr_debug("tx_loop: %d total_tx %d\n",
1304                                 tx_loop, total_tx);
1305                         return tx_loop;
1306                 }
1307                 total_tx += tx_loop;
1308                 pr_debug("tx_loop: %d, total_tx: %d, data: %d\n",
1309                                         tx_loop, total_tx, data);
1310         }
1311
1312         return total_tx;
1313 }
1314
1315 void iscsit_collect_login_stats(
1316         struct iscsi_conn *conn,
1317         u8 status_class,
1318         u8 status_detail)
1319 {
1320         struct iscsi_param *intrname = NULL;
1321         struct iscsi_tiqn *tiqn;
1322         struct iscsi_login_stats *ls;
1323
1324         tiqn = iscsit_snmp_get_tiqn(conn);
1325         if (!tiqn)
1326                 return;
1327
1328         ls = &tiqn->login_stats;
1329
1330         spin_lock(&ls->lock);
1331         if (status_class == ISCSI_STATUS_CLS_SUCCESS)
1332                 ls->accepts++;
1333         else if (status_class == ISCSI_STATUS_CLS_REDIRECT) {
1334                 ls->redirects++;
1335                 ls->last_fail_type = ISCSI_LOGIN_FAIL_REDIRECT;
1336         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1337                  (status_detail == ISCSI_LOGIN_STATUS_AUTH_FAILED)) {
1338                 ls->authenticate_fails++;
1339                 ls->last_fail_type =  ISCSI_LOGIN_FAIL_AUTHENTICATE;
1340         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1341                  (status_detail == ISCSI_LOGIN_STATUS_TGT_FORBIDDEN)) {
1342                 ls->authorize_fails++;
1343                 ls->last_fail_type = ISCSI_LOGIN_FAIL_AUTHORIZE;
1344         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR) &&
1345                  (status_detail == ISCSI_LOGIN_STATUS_INIT_ERR)) {
1346                 ls->negotiate_fails++;
1347                 ls->last_fail_type = ISCSI_LOGIN_FAIL_NEGOTIATE;
1348         } else {
1349                 ls->other_fails++;
1350                 ls->last_fail_type = ISCSI_LOGIN_FAIL_OTHER;
1351         }
1352
1353         /* Save initiator name, ip address and time, if it is a failed login */
1354         if (status_class != ISCSI_STATUS_CLS_SUCCESS) {
1355                 if (conn->param_list)
1356                         intrname = iscsi_find_param_from_key(INITIATORNAME,
1357                                                              conn->param_list);
1358                 strlcpy(ls->last_intr_fail_name,
1359                        (intrname ? intrname->value : "Unknown"),
1360                        sizeof(ls->last_intr_fail_name));
1361
1362                 ls->last_intr_fail_ip_family = conn->login_family;
1363
1364                 ls->last_intr_fail_sockaddr = conn->login_sockaddr;
1365                 ls->last_fail_time = get_jiffies_64();
1366         }
1367
1368         spin_unlock(&ls->lock);
1369 }
1370
1371 struct iscsi_tiqn *iscsit_snmp_get_tiqn(struct iscsi_conn *conn)
1372 {
1373         struct iscsi_portal_group *tpg;
1374
1375         if (!conn)
1376                 return NULL;
1377
1378         tpg = conn->tpg;
1379         if (!tpg)
1380                 return NULL;
1381
1382         if (!tpg->tpg_tiqn)
1383                 return NULL;
1384
1385         return tpg->tpg_tiqn;
1386 }
1387
1388 void iscsit_fill_cxn_timeout_err_stats(struct iscsi_session *sess)
1389 {
1390         struct iscsi_portal_group *tpg = sess->tpg;
1391         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
1392
1393         if (!tiqn)
1394                 return;
1395
1396         spin_lock_bh(&tiqn->sess_err_stats.lock);
1397         strlcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
1398                         sess->sess_ops->InitiatorName,
1399                         sizeof(tiqn->sess_err_stats.last_sess_fail_rem_name));
1400         tiqn->sess_err_stats.last_sess_failure_type =
1401                         ISCSI_SESS_ERR_CXN_TIMEOUT;
1402         tiqn->sess_err_stats.cxn_timeout_errors++;
1403         atomic_long_inc(&sess->conn_timeout_errors);
1404         spin_unlock_bh(&tiqn->sess_err_stats.lock);
1405 }