drivers/staging/lustre/lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/selftest/conctl.c
  33  *
  34  * Infrastructure of LST console
  35  *
  36  * Author: Liang Zhen <liangzhen@clusterfs.com>
  37  */
  38
  39 #include <linux/libcfs/libcfs.h>
  40 #include <linux/lnet/lib-lnet.h>
  41 #include "console.h"
  42 #include "conrpc.h"
  43
  44 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  45 do {                                                    \
  46         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  47                 (p)->nle_nactive++;                     \
  48         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  49                 (p)->nle_nbusy++;                       \
  50         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  51                 (p)->nle_ndown++;                       \
  52         else                                            \
  53                 (p)->nle_nunknown++;                    \
  54         (p)->nle_nnode++;                               \
  55 } while (0)
  56
  57 struct lstcon_session console_session;
  58
  59 static void
  60 lstcon_node_get(struct lstcon_node *nd)
  61 {
  62         LASSERT(nd->nd_ref >= 1);
  63
  64         nd->nd_ref++;
  65 }
  66
  67 static int
  68 lstcon_node_find(struct lnet_process_id id, struct lstcon_node **ndpp,
  69                  int create)
  70 {
  71         struct lstcon_ndlink    *ndl;
  72         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  73
  74         LASSERT(id.nid != LNET_NID_ANY);
  75
  76         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
  77                             ndl_hlink) {
  78                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  79                     ndl->ndl_node->nd_id.pid != id.pid)
  80                         continue;
  81
  82                 lstcon_node_get(ndl->ndl_node);
  83                 *ndpp = ndl->ndl_node;
  84                 return 0;
  85         }
  86
  87         if (!create)
  88                 return -ENOENT;
  89
  90         LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
  91         if (!*ndpp)
  92                 return -ENOMEM;
  93
  94         ndl = (struct lstcon_ndlink *)(*ndpp + 1);
  95
  96         ndl->ndl_node = *ndpp;
  97
  98         ndl->ndl_node->nd_ref = 1;
  99         ndl->ndl_node->nd_id = id;
 100         ndl->ndl_node->nd_stamp = cfs_time_current();
 101         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 102         ndl->ndl_node->nd_timeout = 0;
 103         memset(&ndl->ndl_node->nd_ping, 0, sizeof(struct lstcon_rpc));
 104
 105         /*
 106          * queued in global hash & list, no refcount is taken by
 107          * global hash & list, if caller release his refcount,
 108          * node will be released
 109          */
 110         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 111         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 112
 113         return 0;
 114 }
 115
 116 static void
 117 lstcon_node_put(struct lstcon_node *nd)
 118 {
 119         struct lstcon_ndlink *ndl;
 120
 121         LASSERT(nd->nd_ref > 0);
 122
 123         if (--nd->nd_ref > 0)
 124                 return;
 125
 126         ndl = (struct lstcon_ndlink *)(nd + 1);
 127
 128         LASSERT(!list_empty(&ndl->ndl_link));
 129         LASSERT(!list_empty(&ndl->ndl_hlink));
 130
 131         /* remove from session */
 132         list_del(&ndl->ndl_link);
 133         list_del(&ndl->ndl_hlink);
 134
 135         LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 136 }
 137
 138 static int
 139 lstcon_ndlink_find(struct list_head *hash, struct lnet_process_id id,
 140                    struct lstcon_ndlink **ndlpp, int create)
 141 {
 142         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 143         struct lstcon_ndlink *ndl;
 144         struct lstcon_node *nd;
 145         int rc;
 146
 147         if (id.nid == LNET_NID_ANY)
 148                 return -EINVAL;
 149
 150         /* search in hash */
 151         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 152                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 153                     ndl->ndl_node->nd_id.pid != id.pid)
 154                         continue;
 155
 156                 *ndlpp = ndl;
 157                 return 0;
 158         }
 159
 160         if (!create)
 161                 return -ENOENT;
 162
 163         /* find or create in session hash */
 164         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 165         if (rc)
 166                 return rc;
 167
 168         LIBCFS_ALLOC(ndl, sizeof(struct lstcon_ndlink));
 169         if (!ndl) {
 170                 lstcon_node_put(nd);
 171                 return -ENOMEM;
 172         }
 173
 174         *ndlpp = ndl;
 175
 176         ndl->ndl_node = nd;
 177         INIT_LIST_HEAD(&ndl->ndl_link);
 178         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 179
 180         return 0;
 181 }
 182
 183 static void
 184 lstcon_ndlink_release(struct lstcon_ndlink *ndl)
 185 {
 186         LASSERT(list_empty(&ndl->ndl_link));
 187         LASSERT(!list_empty(&ndl->ndl_hlink));
 188
 189         list_del(&ndl->ndl_hlink); /* delete from hash */
 190         lstcon_node_put(ndl->ndl_node);
 191
 192         LIBCFS_FREE(ndl, sizeof(*ndl));
 193 }
 194
 195 static int
 196 lstcon_group_alloc(char *name, struct lstcon_group **grpp)
 197 {
 198         struct lstcon_group *grp;
 199         int i;
 200
 201         LIBCFS_ALLOC(grp, offsetof(struct lstcon_group,
 202                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 203         if (!grp)
 204                 return -ENOMEM;
 205
 206         grp->grp_ref = 1;
 207         if (name) {
 208                 if (strlen(name) > sizeof(grp->grp_name) - 1) {
 209                         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 210                                     grp_ndl_hash[LST_NODE_HASHSIZE]));
 211                         return -E2BIG;
 212                 }
 213                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
 214         }
 215
 216         INIT_LIST_HEAD(&grp->grp_link);
 217         INIT_LIST_HEAD(&grp->grp_ndl_list);
 218         INIT_LIST_HEAD(&grp->grp_trans_list);
 219
 220         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 221                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 222
 223         *grpp = grp;
 224
 225         return 0;
 226 }
 227
 228 static void
 229 lstcon_group_addref(struct lstcon_group *grp)
 230 {
 231         grp->grp_ref++;
 232 }
 233
 234 static void lstcon_group_ndlink_release(struct lstcon_group *,
 235                                         struct lstcon_ndlink *);
 236
 237 static void
 238 lstcon_group_drain(struct lstcon_group *grp, int keep)
 239 {
 240         struct lstcon_ndlink *ndl;
 241         struct lstcon_ndlink *tmp;
 242
 243         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 244                 if (!(ndl->ndl_node->nd_state & keep))
 245                         lstcon_group_ndlink_release(grp, ndl);
 246         }
 247 }
 248
 249 static void
 250 lstcon_group_decref(struct lstcon_group *grp)
 251 {
 252         int i;
 253
 254         if (--grp->grp_ref > 0)
 255                 return;
 256
 257         if (!list_empty(&grp->grp_link))
 258                 list_del(&grp->grp_link);
 259
 260         lstcon_group_drain(grp, 0);
 261
 262         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 263                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 264
 265         LIBCFS_FREE(grp, offsetof(struct lstcon_group,
 266                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 267 }
 268
 269 static int
 270 lstcon_group_find(const char *name, struct lstcon_group **grpp)
 271 {
 272         struct lstcon_group *grp;
 273
 274         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 275                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE))
 276                         continue;
 277
 278                 lstcon_group_addref(grp); /* +1 ref for caller */
 279                 *grpp = grp;
 280                 return 0;
 281         }
 282
 283         return -ENOENT;
 284 }
 285
 286 static int
 287 lstcon_group_ndlink_find(struct lstcon_group *grp, struct lnet_process_id id,
 288                          struct lstcon_ndlink **ndlpp, int create)
 289 {
 290         int rc;
 291
 292         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 293         if (rc)
 294                 return rc;
 295
 296         if (!list_empty(&(*ndlpp)->ndl_link))
 297                 return 0;
 298
 299         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 300         grp->grp_nnode++;
 301
 302         return 0;
 303 }
 304
 305 static void
 306 lstcon_group_ndlink_release(struct lstcon_group *grp, struct lstcon_ndlink *ndl)
 307 {
 308         list_del_init(&ndl->ndl_link);
 309         lstcon_ndlink_release(ndl);
 310         grp->grp_nnode--;
 311 }
 312
 313 static void
 314 lstcon_group_ndlink_move(struct lstcon_group *old,
 315                          struct lstcon_group *new, struct lstcon_ndlink *ndl)
 316 {
 317         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 318                                         LST_NODE_HASHSIZE;
 319
 320         list_del(&ndl->ndl_hlink);
 321         list_del(&ndl->ndl_link);
 322         old->grp_nnode--;
 323
 324         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 325         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 326         new->grp_nnode++;
 327 }
 328
 329 static void
 330 lstcon_group_move(struct lstcon_group *old, struct lstcon_group *new)
 331 {
 332         struct lstcon_ndlink *ndl;
 333
 334         while (!list_empty(&old->grp_ndl_list)) {
 335                 ndl = list_entry(old->grp_ndl_list.next,
 336                                  struct lstcon_ndlink, ndl_link);
 337                 lstcon_group_ndlink_move(old, new, ndl);
 338         }
 339 }
 340
 341 static int
 342 lstcon_sesrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 343 {
 344         struct lstcon_group *grp = (struct lstcon_group *)arg;
 345
 346         switch (transop) {
 347         case LST_TRANS_SESNEW:
 348                 if (nd->nd_state == LST_NODE_ACTIVE)
 349                         return 0;
 350                 break;
 351
 352         case LST_TRANS_SESEND:
 353                 if (nd->nd_state != LST_NODE_ACTIVE)
 354                         return 0;
 355
 356                 if (grp && nd->nd_ref > 1)
 357                         return 0;
 358                 break;
 359
 360         case LST_TRANS_SESQRY:
 361                 break;
 362
 363         default:
 364                 LBUG();
 365         }
 366
 367         return 1;
 368 }
 369
 370 static int
 371 lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 372                       struct lstcon_rpc_ent __user *ent_up)
 373 {
 374         struct srpc_debug_reply *rep;
 375
 376         switch (transop) {
 377         case LST_TRANS_SESNEW:
 378         case LST_TRANS_SESEND:
 379                 return 0;
 380
 381         case LST_TRANS_SESQRY:
 382                 rep = &msg->msg_body.dbg_reply;
 383
 384                 if (copy_to_user(&ent_up->rpe_priv[0],
 385                                  &rep->dbg_timeout, sizeof(int)) ||
 386                     copy_to_user(&ent_up->rpe_payload[0],
 387                                  &rep->dbg_name, LST_NAME_SIZE))
 388                         return -EFAULT;
 389
 390                 return 0;
 391
 392         default:
 393                 LBUG();
 394         }
 395
 396         return 0;
 397 }
 398
 399 static int
 400 lstcon_group_nodes_add(struct lstcon_group *grp,
 401                        int count, struct lnet_process_id __user *ids_up,
 402                        unsigned int *featp,
 403                        struct list_head __user *result_up)
 404 {
 405         struct lstcon_rpc_trans *trans;
 406         struct lstcon_ndlink    *ndl;
 407         struct lstcon_group *tmp;
 408         struct lnet_process_id id;
 409         int i;
 410         int rc;
 411
 412         rc = lstcon_group_alloc(NULL, &tmp);
 413         if (rc) {
 414                 CERROR("Out of memory\n");
 415                 return -ENOMEM;
 416         }
 417
 418         for (i = 0 ; i < count; i++) {
 419                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 420                         rc = -EFAULT;
 421                         break;
 422                 }
 423
 424                 /* skip if it's in this group already */
 425                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 426                 if (!rc)
 427                         continue;
 428
 429                 /* add to tmp group */
 430                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 431                 if (rc) {
 432                         CERROR("Can't create ndlink, out of memory\n");
 433                         break;
 434                 }
 435         }
 436
 437         if (rc) {
 438                 lstcon_group_decref(tmp);
 439                 return rc;
 440         }
 441
 442         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 443                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 444                                      tmp, lstcon_sesrpc_condition, &trans);
 445         if (rc) {
 446                 CERROR("Can't create transaction: %d\n", rc);
 447                 lstcon_group_decref(tmp);
 448                 return rc;
 449         }
 450
 451         /* post all RPCs */
 452         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 453
 454         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 455                                           lstcon_sesrpc_readent);
 456         *featp = trans->tas_features;
 457
 458         /* destroy all RPGs */
 459         lstcon_rpc_trans_destroy(trans);
 460
 461         lstcon_group_move(tmp, grp);
 462         lstcon_group_decref(tmp);
 463
 464         return rc;
 465 }
 466
 467 static int
 468 lstcon_group_nodes_remove(struct lstcon_group *grp,
 469                           int count, struct lnet_process_id __user *ids_up,
 470                           struct list_head __user *result_up)
 471 {
 472         struct lstcon_rpc_trans *trans;
 473         struct lstcon_ndlink *ndl;
 474         struct lstcon_group *tmp;
 475         struct lnet_process_id id;
 476         int rc;
 477         int i;
 478
 479         /* End session and remove node from the group */
 480
 481         rc = lstcon_group_alloc(NULL, &tmp);
 482         if (rc) {
 483                 CERROR("Out of memory\n");
 484                 return -ENOMEM;
 485         }
 486
 487         for (i = 0; i < count; i++) {
 488                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 489                         rc = -EFAULT;
 490                         goto error;
 491                 }
 492
 493                 /* move node to tmp group */
 494                 if (!lstcon_group_ndlink_find(grp, id, &ndl, 0))
 495                         lstcon_group_ndlink_move(grp, tmp, ndl);
 496         }
 497
 498         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 499                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 500                                      tmp, lstcon_sesrpc_condition, &trans);
 501         if (rc) {
 502                 CERROR("Can't create transaction: %d\n", rc);
 503                 goto error;
 504         }
 505
 506         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 507
 508         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 509
 510         lstcon_rpc_trans_destroy(trans);
 511         /* release nodes anyway, because we can't rollback status */
 512         lstcon_group_decref(tmp);
 513
 514         return rc;
 515 error:
 516         lstcon_group_move(tmp, grp);
 517         lstcon_group_decref(tmp);
 518
 519         return rc;
 520 }
 521
 522 int
 523 lstcon_group_add(char *name)
 524 {
 525         struct lstcon_group *grp;
 526         int rc;
 527
 528         rc = lstcon_group_find(name, &grp) ? 0 : -EEXIST;
 529         if (rc) {
 530                 /* find a group with same name */
 531                 lstcon_group_decref(grp);
 532                 return rc;
 533         }
 534
 535         rc = lstcon_group_alloc(name, &grp);
 536         if (rc) {
 537                 CERROR("Can't allocate descriptor for group %s\n", name);
 538                 return -ENOMEM;
 539         }
 540
 541         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 542
 543         return rc;
 544 }
 545
 546 int
 547 lstcon_nodes_add(char *name, int count, struct lnet_process_id __user *ids_up,
 548                  unsigned int *featp, struct list_head __user *result_up)
 549 {
 550         struct lstcon_group *grp;
 551         int rc;
 552
 553         LASSERT(count > 0);
 554         LASSERT(ids_up);
 555
 556         rc = lstcon_group_find(name, &grp);
 557         if (rc) {
 558                 CDEBUG(D_NET, "Can't find group %s\n", name);
 559                 return rc;
 560         }
 561
 562         if (grp->grp_ref > 2) {
 563                 /* referred by other threads or test */
 564                 CDEBUG(D_NET, "Group %s is busy\n", name);
 565                 lstcon_group_decref(grp);
 566
 567                 return -EBUSY;
 568         }
 569
 570         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 571
 572         lstcon_group_decref(grp);
 573
 574         return rc;
 575 }
 576
 577 int
 578 lstcon_group_del(char *name)
 579 {
 580         struct lstcon_rpc_trans *trans;
 581         struct lstcon_group *grp;
 582         int rc;
 583
 584         rc = lstcon_group_find(name, &grp);
 585         if (rc) {
 586                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 587                 return rc;
 588         }
 589
 590         if (grp->grp_ref > 2) {
 591                 /* referred by others threads or test */
 592                 CDEBUG(D_NET, "Group %s is busy\n", name);
 593                 lstcon_group_decref(grp);
 594                 return -EBUSY;
 595         }
 596
 597         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 598                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 599                                      grp, lstcon_sesrpc_condition, &trans);
 600         if (rc) {
 601                 CERROR("Can't create transaction: %d\n", rc);
 602                 lstcon_group_decref(grp);
 603                 return rc;
 604         }
 605
 606         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 607
 608         lstcon_rpc_trans_destroy(trans);
 609
 610         lstcon_group_decref(grp);
 611         /*
 612          * -ref for session, it's destroyed,
 613          * status can't be rolled back, destroy group anyway
 614          */
 615         lstcon_group_decref(grp);
 616
 617         return rc;
 618 }
 619
 620 int
 621 lstcon_group_clean(char *name, int args)
 622 {
 623         struct lstcon_group *grp = NULL;
 624         int rc;
 625
 626         rc = lstcon_group_find(name, &grp);
 627         if (rc) {
 628                 CDEBUG(D_NET, "Can't find group %s\n", name);
 629                 return rc;
 630         }
 631
 632         if (grp->grp_ref > 2) {
 633                 /* referred by test */
 634                 CDEBUG(D_NET, "Group %s is busy\n", name);
 635                 lstcon_group_decref(grp);
 636                 return -EBUSY;
 637         }
 638
 639         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 640                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 641
 642         lstcon_group_drain(grp, args);
 643
 644         lstcon_group_decref(grp);
 645         /* release empty group */
 646         if (list_empty(&grp->grp_ndl_list))
 647                 lstcon_group_decref(grp);
 648
 649         return 0;
 650 }
 651
 652 int
 653 lstcon_nodes_remove(char *name, int count,
 654                     struct lnet_process_id __user *ids_up,
 655                     struct list_head __user *result_up)
 656 {
 657         struct lstcon_group *grp = NULL;
 658         int rc;
 659
 660         rc = lstcon_group_find(name, &grp);
 661         if (rc) {
 662                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 663                 return rc;
 664         }
 665
 666         if (grp->grp_ref > 2) {
 667                 /* referred by test */
 668                 CDEBUG(D_NET, "Group %s is busy\n", name);
 669                 lstcon_group_decref(grp);
 670                 return -EBUSY;
 671         }
 672
 673         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 674
 675         lstcon_group_decref(grp);
 676         /* release empty group */
 677         if (list_empty(&grp->grp_ndl_list))
 678                 lstcon_group_decref(grp);
 679
 680         return rc;
 681 }
 682
 683 int
 684 lstcon_group_refresh(char *name, struct list_head __user *result_up)
 685 {
 686         struct lstcon_rpc_trans *trans;
 687         struct lstcon_group *grp;
 688         int rc;
 689
 690         rc = lstcon_group_find(name, &grp);
 691         if (rc) {
 692                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 693                 return rc;
 694         }
 695
 696         if (grp->grp_ref > 2) {
 697                 /* referred by test */
 698                 CDEBUG(D_NET, "Group %s is busy\n", name);
 699                 lstcon_group_decref(grp);
 700                 return -EBUSY;
 701         }
 702
 703         /* re-invite all inactive nodes int the group */
 704         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 705                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 706                                      grp, lstcon_sesrpc_condition, &trans);
 707         if (rc) {
 708                 /* local error, return */
 709                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 710                 lstcon_group_decref(grp);
 711                 return rc;
 712         }
 713
 714         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 715
 716         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 717
 718         lstcon_rpc_trans_destroy(trans);
 719         /* -ref for me */
 720         lstcon_group_decref(grp);
 721
 722         return rc;
 723 }
 724
 725 int
 726 lstcon_group_list(int index, int len, char __user *name_up)
 727 {
 728         struct lstcon_group *grp;
 729
 730         LASSERT(index >= 0);
 731         LASSERT(name_up);
 732
 733         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 734                 if (!index--) {
 735                         return copy_to_user(name_up, grp->grp_name, len) ?
 736                                             -EFAULT : 0;
 737                 }
 738         }
 739
 740         return -ENOENT;
 741 }
 742
 743 static int
 744 lstcon_nodes_getent(struct list_head *head, int *index_p,
 745                     int *count_p, struct lstcon_node_ent __user *dents_up)
 746 {
 747         struct lstcon_ndlink *ndl;
 748         struct lstcon_node *nd;
 749         int count = 0;
 750         int index = 0;
 751
 752         LASSERT(index_p && count_p);
 753         LASSERT(dents_up);
 754         LASSERT(*index_p >= 0);
 755         LASSERT(*count_p > 0);
 756
 757         list_for_each_entry(ndl, head, ndl_link) {
 758                 if (index++ < *index_p)
 759                         continue;
 760
 761                 if (count >= *count_p)
 762                         break;
 763
 764                 nd = ndl->ndl_node;
 765                 if (copy_to_user(&dents_up[count].nde_id,
 766                                  &nd->nd_id, sizeof(nd->nd_id)) ||
 767                     copy_to_user(&dents_up[count].nde_state,
 768                                  &nd->nd_state, sizeof(nd->nd_state)))
 769                         return -EFAULT;
 770
 771                 count++;
 772         }
 773
 774         if (index <= *index_p)
 775                 return -ENOENT;
 776
 777         *count_p = count;
 778         *index_p = index;
 779
 780         return 0;
 781 }
 782
 783 int
 784 lstcon_group_info(char *name, struct lstcon_ndlist_ent __user *gents_p,
 785                   int *index_p, int *count_p,
 786                   struct lstcon_node_ent __user *dents_up)
 787 {
 788         struct lstcon_ndlist_ent *gentp;
 789         struct lstcon_group *grp;
 790         struct lstcon_ndlink *ndl;
 791         int rc;
 792
 793         rc = lstcon_group_find(name, &grp);
 794         if (rc) {
 795                 CDEBUG(D_NET, "Can't find group %s\n", name);
 796                 return rc;
 797         }
 798
 799         if (dents_up) {
 800                 /* verbose query */
 801                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 802                                          index_p, count_p, dents_up);
 803                 lstcon_group_decref(grp);
 804
 805                 return rc;
 806         }
 807
 808         /* non-verbose query */
 809         LIBCFS_ALLOC(gentp, sizeof(struct lstcon_ndlist_ent));
 810         if (!gentp) {
 811                 CERROR("Can't allocate ndlist_ent\n");
 812                 lstcon_group_decref(grp);
 813
 814                 return -ENOMEM;
 815         }
 816
 817         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 818                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 819
 820         rc = copy_to_user(gents_p, gentp,
 821                           sizeof(struct lstcon_ndlist_ent)) ? -EFAULT : 0;
 822
 823         LIBCFS_FREE(gentp, sizeof(struct lstcon_ndlist_ent));
 824
 825         lstcon_group_decref(grp);
 826
 827         return rc;
 828 }
 829
 830 static int
 831 lstcon_batch_find(const char *name, struct lstcon_batch **batpp)
 832 {
 833         struct lstcon_batch *bat;
 834
 835         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 836                 if (!strncmp(bat->bat_name, name, LST_NAME_SIZE)) {
 837                         *batpp = bat;
 838                         return 0;
 839                 }
 840         }
 841
 842         return -ENOENT;
 843 }
 844
 845 int
 846 lstcon_batch_add(char *name)
 847 {
 848         struct lstcon_batch *bat;
 849         int i;
 850         int rc;
 851
 852         rc = !lstcon_batch_find(name, &bat) ? -EEXIST : 0;
 853         if (rc) {
 854                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 855                 return rc;
 856         }
 857
 858         LIBCFS_ALLOC(bat, sizeof(struct lstcon_batch));
 859         if (!bat) {
 860                 CERROR("Can't allocate descriptor for batch %s\n", name);
 861                 return -ENOMEM;
 862         }
 863
 864         LIBCFS_ALLOC(bat->bat_cli_hash,
 865                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 866         if (!bat->bat_cli_hash) {
 867                 CERROR("Can't allocate hash for batch %s\n", name);
 868                 LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
 869
 870                 return -ENOMEM;
 871         }
 872
 873         LIBCFS_ALLOC(bat->bat_srv_hash,
 874                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 875         if (!bat->bat_srv_hash) {
 876                 CERROR("Can't allocate hash for batch %s\n", name);
 877                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 878                 LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
 879
 880                 return -ENOMEM;
 881         }
 882
 883         if (strlen(name) > sizeof(bat->bat_name) - 1) {
 884                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
 885                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 886                 LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
 887                 return -E2BIG;
 888         }
 889         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
 890         bat->bat_hdr.tsb_index = 0;
 891         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 892
 893         bat->bat_ntest = 0;
 894         bat->bat_state = LST_BATCH_IDLE;
 895
 896         INIT_LIST_HEAD(&bat->bat_cli_list);
 897         INIT_LIST_HEAD(&bat->bat_srv_list);
 898         INIT_LIST_HEAD(&bat->bat_test_list);
 899         INIT_LIST_HEAD(&bat->bat_trans_list);
 900
 901         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 902                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 903                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 904         }
 905
 906         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 907
 908         return rc;
 909 }
 910
 911 int
 912 lstcon_batch_list(int index, int len, char __user *name_up)
 913 {
 914         struct lstcon_batch *bat;
 915
 916         LASSERT(name_up);
 917         LASSERT(index >= 0);
 918
 919         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 920                 if (!index--) {
 921                         return copy_to_user(name_up, bat->bat_name, len) ?
 922                                             -EFAULT : 0;
 923                 }
 924         }
 925
 926         return -ENOENT;
 927 }
 928
 929 int
 930 lstcon_batch_info(char *name, struct lstcon_test_batch_ent __user *ent_up,
 931                   int server, int testidx, int *index_p, int *ndent_p,
 932                   struct lstcon_node_ent __user *dents_up)
 933 {
 934         struct lstcon_test_batch_ent *entp;
 935         struct list_head *clilst;
 936         struct list_head *srvlst;
 937         struct lstcon_test *test = NULL;
 938         struct lstcon_batch *bat;
 939         struct lstcon_ndlink    *ndl;
 940         int rc;
 941
 942         rc = lstcon_batch_find(name, &bat);
 943         if (rc) {
 944                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 945                 return -ENOENT;
 946         }
 947
 948         if (testidx > 0) {
 949                 /* query test, test index start from 1 */
 950                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 951                         if (testidx-- == 1)
 952                                 break;
 953                 }
 954
 955                 if (testidx > 0) {
 956                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 957                         return -ENOENT;
 958                 }
 959         }
 960
 961         clilst = !test ? &bat->bat_cli_list :
 962                          &test->tes_src_grp->grp_ndl_list;
 963         srvlst = !test ? &bat->bat_srv_list :
 964                          &test->tes_dst_grp->grp_ndl_list;
 965
 966         if (dents_up) {
 967                 rc = lstcon_nodes_getent((server ? srvlst : clilst),
 968                                          index_p, ndent_p, dents_up);
 969                 return rc;
 970         }
 971
 972         /* non-verbose query */
 973         LIBCFS_ALLOC(entp, sizeof(struct lstcon_test_batch_ent));
 974         if (!entp)
 975                 return -ENOMEM;
 976
 977         if (!test) {
 978                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 979                 entp->u.tbe_batch.bae_state = bat->bat_state;
 980         } else {
 981                 entp->u.tbe_test.tse_type = test->tes_type;
 982                 entp->u.tbe_test.tse_loop = test->tes_loop;
 983                 entp->u.tbe_test.tse_concur = test->tes_concur;
 984         }
 985
 986         list_for_each_entry(ndl, clilst, ndl_link)
 987                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 988
 989         list_for_each_entry(ndl, srvlst, ndl_link)
 990                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 991
 992         rc = copy_to_user(ent_up, entp,
 993                           sizeof(struct lstcon_test_batch_ent)) ? -EFAULT : 0;
 994
 995         LIBCFS_FREE(entp, sizeof(struct lstcon_test_batch_ent));
 996
 997         return rc;
 998 }
 999
1000 static int
1001 lstcon_batrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1002 {
1003         switch (transop) {
1004         case LST_TRANS_TSBRUN:
1005                 if (nd->nd_state != LST_NODE_ACTIVE)
1006                         return -ENETDOWN;
1007                 break;
1008
1009         case LST_TRANS_TSBSTOP:
1010                 if (nd->nd_state != LST_NODE_ACTIVE)
1011                         return 0;
1012                 break;
1013
1014         case LST_TRANS_TSBCLIQRY:
1015         case LST_TRANS_TSBSRVQRY:
1016                 break;
1017         }
1018
1019         return 1;
1020 }
1021
1022 static int
1023 lstcon_batch_op(struct lstcon_batch *bat, int transop,
1024                 struct list_head __user *result_up)
1025 {
1026         struct lstcon_rpc_trans *trans;
1027         int rc;
1028
1029         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1030                                      &bat->bat_trans_list, transop,
1031                                      bat, lstcon_batrpc_condition, &trans);
1032         if (rc) {
1033                 CERROR("Can't create transaction: %d\n", rc);
1034                 return rc;
1035         }
1036
1037         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1038
1039         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1040
1041         lstcon_rpc_trans_destroy(trans);
1042
1043         return rc;
1044 }
1045
1046 int
1047 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1048 {
1049         struct lstcon_batch *bat;
1050         int rc;
1051
1052         if (lstcon_batch_find(name, &bat)) {
1053                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1054                 return -ENOENT;
1055         }
1056
1057         bat->bat_arg = timeout;
1058
1059         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1060
1061         /* mark batch as running if it's started in any node */
1062         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0))
1063                 bat->bat_state = LST_BATCH_RUNNING;
1064
1065         return rc;
1066 }
1067
1068 int
1069 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1070 {
1071         struct lstcon_batch *bat;
1072         int rc;
1073
1074         if (lstcon_batch_find(name, &bat)) {
1075                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1076                 return -ENOENT;
1077         }
1078
1079         bat->bat_arg = force;
1080
1081         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1082
1083         /* mark batch as stopped if all RPCs finished */
1084         if (!lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0))
1085                 bat->bat_state = LST_BATCH_IDLE;
1086
1087         return rc;
1088 }
1089
1090 static void
1091 lstcon_batch_destroy(struct lstcon_batch *bat)
1092 {
1093         struct lstcon_ndlink *ndl;
1094         struct lstcon_test *test;
1095         int i;
1096
1097         list_del(&bat->bat_link);
1098
1099         while (!list_empty(&bat->bat_test_list)) {
1100                 test = list_entry(bat->bat_test_list.next,
1101                                   struct lstcon_test, tes_link);
1102                 LASSERT(list_empty(&test->tes_trans_list));
1103
1104                 list_del(&test->tes_link);
1105
1106                 lstcon_group_decref(test->tes_src_grp);
1107                 lstcon_group_decref(test->tes_dst_grp);
1108
1109                 LIBCFS_FREE(test, offsetof(struct lstcon_test,
1110                                            tes_param[test->tes_paramlen]));
1111         }
1112
1113         LASSERT(list_empty(&bat->bat_trans_list));
1114
1115         while (!list_empty(&bat->bat_cli_list)) {
1116                 ndl = list_entry(bat->bat_cli_list.next,
1117                                  struct lstcon_ndlink, ndl_link);
1118                 list_del_init(&ndl->ndl_link);
1119
1120                 lstcon_ndlink_release(ndl);
1121         }
1122
1123         while (!list_empty(&bat->bat_srv_list)) {
1124                 ndl = list_entry(bat->bat_srv_list.next,
1125                                  struct lstcon_ndlink, ndl_link);
1126                 list_del_init(&ndl->ndl_link);
1127
1128                 lstcon_ndlink_release(ndl);
1129         }
1130
1131         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1132                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1133                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1134         }
1135
1136         LIBCFS_FREE(bat->bat_cli_hash,
1137                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1138         LIBCFS_FREE(bat->bat_srv_hash,
1139                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1140         LIBCFS_FREE(bat, sizeof(struct lstcon_batch));
1141 }
1142
1143 static int
1144 lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
1145 {
1146         struct lstcon_test *test;
1147         struct lstcon_batch *batch;
1148         struct lstcon_ndlink *ndl;
1149         struct list_head *hash;
1150         struct list_head *head;
1151
1152         test = (struct lstcon_test *)arg;
1153         LASSERT(test);
1154
1155         batch = test->tes_batch;
1156         LASSERT(batch);
1157
1158         if (test->tes_oneside &&
1159             transop == LST_TRANS_TSBSRVADD)
1160                 return 0;
1161
1162         if (nd->nd_state != LST_NODE_ACTIVE)
1163                 return -ENETDOWN;
1164
1165         if (transop == LST_TRANS_TSBCLIADD) {
1166                 hash = batch->bat_cli_hash;
1167                 head = &batch->bat_cli_list;
1168
1169         } else {
1170                 LASSERT(transop == LST_TRANS_TSBSRVADD);
1171
1172                 hash = batch->bat_srv_hash;
1173                 head = &batch->bat_srv_list;
1174         }
1175
1176         LASSERT(nd->nd_id.nid != LNET_NID_ANY);
1177
1178         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1))
1179                 return -ENOMEM;
1180
1181         if (list_empty(&ndl->ndl_link))
1182                 list_add_tail(&ndl->ndl_link, head);
1183
1184         return 1;
1185 }
1186
1187 static int
1188 lstcon_test_nodes_add(struct lstcon_test *test,
1189                       struct list_head __user *result_up)
1190 {
1191         struct lstcon_rpc_trans *trans;
1192         struct lstcon_group *grp;
1193         int transop;
1194         int rc;
1195
1196         LASSERT(test->tes_src_grp);
1197         LASSERT(test->tes_dst_grp);
1198
1199         transop = LST_TRANS_TSBSRVADD;
1200         grp = test->tes_dst_grp;
1201 again:
1202         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1203                                      &test->tes_trans_list, transop,
1204                                      test, lstcon_testrpc_condition, &trans);
1205         if (rc) {
1206                 CERROR("Can't create transaction: %d\n", rc);
1207                 return rc;
1208         }
1209
1210         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1211
1212         if (lstcon_trans_stat()->trs_rpc_errno ||
1213             lstcon_trans_stat()->trs_fwk_errno) {
1214                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1215
1216                 lstcon_rpc_trans_destroy(trans);
1217                 /* return if any error */
1218                 CDEBUG(D_NET, "Failed to add test %s, RPC error %d, framework error %d\n",
1219                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1220                        lstcon_trans_stat()->trs_rpc_errno,
1221                        lstcon_trans_stat()->trs_fwk_errno);
1222
1223                 return rc;
1224         }
1225
1226         lstcon_rpc_trans_destroy(trans);
1227
1228         if (transop == LST_TRANS_TSBCLIADD)
1229                 return rc;
1230
1231         transop = LST_TRANS_TSBCLIADD;
1232         grp = test->tes_src_grp;
1233         test->tes_cliidx = 0;
1234
1235         /* requests to test clients */
1236         goto again;
1237 }
1238
1239 static int
1240 lstcon_verify_batch(const char *name, struct lstcon_batch **batch)
1241 {
1242         int rc;
1243
1244         rc = lstcon_batch_find(name, batch);
1245         if (rc) {
1246                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1247                 return rc;
1248         }
1249
1250         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1251                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1252                 return -EINVAL;
1253         }
1254
1255         return 0;
1256 }
1257
1258 static int
1259 lstcon_verify_group(const char *name, struct lstcon_group **grp)
1260 {
1261         int rc;
1262         struct lstcon_ndlink    *ndl;
1263
1264         rc = lstcon_group_find(name, grp);
1265         if (rc) {
1266                 CDEBUG(D_NET, "can't find group %s\n", name);
1267                 return rc;
1268         }
1269
1270         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1271                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1272                         return 0;
1273         }
1274
1275         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1276
1277         return -EINVAL;
1278 }
1279
1280 int
1281 lstcon_test_add(char *batch_name, int type, int loop,
1282                 int concur, int dist, int span,
1283                 char *src_name, char *dst_name,
1284                 void *param, int paramlen, int *retp,
1285                 struct list_head __user *result_up)
1286 {
1287         struct lstcon_test *test = NULL;
1288         int rc;
1289         struct lstcon_group *src_grp = NULL;
1290         struct lstcon_group *dst_grp = NULL;
1291         struct lstcon_batch *batch = NULL;
1292
1293         /*
1294          * verify that a batch of the given name exists, and the groups
1295          * that will be part of the batch exist and have at least one
1296          * active node
1297          */
1298         rc = lstcon_verify_batch(batch_name, &batch);
1299         if (rc)
1300                 goto out;
1301
1302         rc = lstcon_verify_group(src_name, &src_grp);
1303         if (rc)
1304                 goto out;
1305
1306         rc = lstcon_verify_group(dst_name, &dst_grp);
1307         if (rc)
1308                 goto out;
1309
1310         if (dst_grp->grp_userland)
1311                 *retp = 1;
1312
1313         LIBCFS_ALLOC(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1314         if (!test) {
1315                 CERROR("Can't allocate test descriptor\n");
1316                 rc = -ENOMEM;
1317
1318                 goto out;
1319         }
1320
1321         test->tes_hdr.tsb_id = batch->bat_hdr.tsb_id;
1322         test->tes_batch = batch;
1323         test->tes_type = type;
1324         test->tes_oneside = 0; /* TODO */
1325         test->tes_loop = loop;
1326         test->tes_concur = concur;
1327         test->tes_stop_onerr = 1; /* TODO */
1328         test->tes_span = span;
1329         test->tes_dist = dist;
1330         test->tes_cliidx = 0; /* just used for creating RPC */
1331         test->tes_src_grp = src_grp;
1332         test->tes_dst_grp = dst_grp;
1333         INIT_LIST_HEAD(&test->tes_trans_list);
1334
1335         if (param) {
1336                 test->tes_paramlen = paramlen;
1337                 memcpy(&test->tes_param[0], param, paramlen);
1338         }
1339
1340         rc = lstcon_test_nodes_add(test, result_up);
1341
1342         if (rc)
1343                 goto out;
1344
1345         if (lstcon_trans_stat()->trs_rpc_errno ||
1346             lstcon_trans_stat()->trs_fwk_errno)
1347                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1348                        batch_name);
1349
1350         /* add to test list anyway, so user can check what's going on */
1351         list_add_tail(&test->tes_link, &batch->bat_test_list);
1352
1353         batch->bat_ntest++;
1354         test->tes_hdr.tsb_index = batch->bat_ntest;
1355
1356         /*  hold groups so nobody can change them */
1357         return rc;
1358 out:
1359         if (test)
1360                 LIBCFS_FREE(test, offsetof(struct lstcon_test, tes_param[paramlen]));
1361
1362         if (dst_grp)
1363                 lstcon_group_decref(dst_grp);
1364
1365         if (src_grp)
1366                 lstcon_group_decref(src_grp);
1367
1368         return rc;
1369 }
1370
1371 static int
1372 lstcon_test_find(struct lstcon_batch *batch, int idx,
1373                  struct lstcon_test **testpp)
1374 {
1375         struct lstcon_test *test;
1376
1377         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1378                 if (idx == test->tes_hdr.tsb_index) {
1379                         *testpp = test;
1380                         return 0;
1381                 }
1382         }
1383
1384         return -ENOENT;
1385 }
1386
1387 static int
1388 lstcon_tsbrpc_readent(int transop, struct srpc_msg *msg,
1389                       struct lstcon_rpc_ent __user *ent_up)
1390 {
1391         struct srpc_batch_reply *rep = &msg->msg_body.bat_reply;
1392
1393         LASSERT(transop == LST_TRANS_TSBCLIQRY ||
1394                 transop == LST_TRANS_TSBSRVQRY);
1395
1396         /* positive errno, framework error code */
1397         if (copy_to_user(&ent_up->rpe_priv[0], &rep->bar_active,
1398                          sizeof(rep->bar_active)))
1399                 return -EFAULT;
1400
1401         return 0;
1402 }
1403
1404 int
1405 lstcon_test_batch_query(char *name, int testidx, int client,
1406                         int timeout, struct list_head __user *result_up)
1407 {
1408         struct lstcon_rpc_trans *trans;
1409         struct list_head *translist;
1410         struct list_head *ndlist;
1411         struct lstcon_tsb_hdr *hdr;
1412         struct lstcon_batch *batch;
1413         struct lstcon_test *test = NULL;
1414         int transop;
1415         int rc;
1416
1417         rc = lstcon_batch_find(name, &batch);
1418         if (rc) {
1419                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1420                 return rc;
1421         }
1422
1423         if (!testidx) {
1424                 translist = &batch->bat_trans_list;
1425                 ndlist = &batch->bat_cli_list;
1426                 hdr = &batch->bat_hdr;
1427         } else {
1428                 /* query specified test only */
1429                 rc = lstcon_test_find(batch, testidx, &test);
1430                 if (rc) {
1431                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1432                         return rc;
1433                 }
1434
1435                 translist = &test->tes_trans_list;
1436                 ndlist = &test->tes_src_grp->grp_ndl_list;
1437                 hdr = &test->tes_hdr;
1438         }
1439
1440         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1441
1442         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1443                                      lstcon_batrpc_condition, &trans);
1444         if (rc) {
1445                 CERROR("Can't create transaction: %d\n", rc);
1446                 return rc;
1447         }
1448
1449         lstcon_rpc_trans_postwait(trans, timeout);
1450
1451         /* query a batch, not a test */
1452         if (!testidx &&
1453             !lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) &&
1454             !lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0)) {
1455                 /* all RPCs finished, and no active test */
1456                 batch->bat_state = LST_BATCH_IDLE;
1457         }
1458
1459         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1460                                           lstcon_tsbrpc_readent);
1461         lstcon_rpc_trans_destroy(trans);
1462
1463         return rc;
1464 }
1465
1466 static int
1467 lstcon_statrpc_readent(int transop, struct srpc_msg *msg,
1468                        struct lstcon_rpc_ent __user *ent_up)
1469 {
1470         struct srpc_stat_reply *rep = &msg->msg_body.stat_reply;
1471         struct sfw_counters __user *sfwk_stat;
1472         struct srpc_counters __user *srpc_stat;
1473         struct lnet_counters __user *lnet_stat;
1474
1475         if (rep->str_status)
1476                 return 0;
1477
1478         sfwk_stat = (struct sfw_counters __user *)&ent_up->rpe_payload[0];
1479         srpc_stat = (struct srpc_counters __user *)(sfwk_stat + 1);
1480         lnet_stat = (struct lnet_counters __user *)(srpc_stat + 1);
1481
1482         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1483             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1484             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1485                 return -EFAULT;
1486
1487         return 0;
1488 }
1489
1490 static int
1491 lstcon_ndlist_stat(struct list_head *ndlist,
1492                    int timeout, struct list_head __user *result_up)
1493 {
1494         struct list_head head;
1495         struct lstcon_rpc_trans *trans;
1496         int rc;
1497
1498         INIT_LIST_HEAD(&head);
1499
1500         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1501                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1502         if (rc) {
1503                 CERROR("Can't create transaction: %d\n", rc);
1504                 return rc;
1505         }
1506
1507         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1508
1509         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1510                                           lstcon_statrpc_readent);
1511         lstcon_rpc_trans_destroy(trans);
1512
1513         return rc;
1514 }
1515
1516 int
1517 lstcon_group_stat(char *grp_name, int timeout,
1518                   struct list_head __user *result_up)
1519 {
1520         struct lstcon_group *grp;
1521         int rc;
1522
1523         rc = lstcon_group_find(grp_name, &grp);
1524         if (rc) {
1525                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1526                 return rc;
1527         }
1528
1529         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1530
1531         lstcon_group_decref(grp);
1532
1533         return rc;
1534 }
1535
1536 int
1537 lstcon_nodes_stat(int count, struct lnet_process_id __user *ids_up,
1538                   int timeout, struct list_head __user *result_up)
1539 {
1540         struct lstcon_ndlink    *ndl;
1541         struct lstcon_group *tmp;
1542         struct lnet_process_id id;
1543         int i;
1544         int rc;
1545
1546         rc = lstcon_group_alloc(NULL, &tmp);
1547         if (rc) {
1548                 CERROR("Out of memory\n");
1549                 return -ENOMEM;
1550         }
1551
1552         for (i = 0 ; i < count; i++) {
1553                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1554                         rc = -EFAULT;
1555                         break;
1556                 }
1557
1558                 /* add to tmp group */
1559                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1560                 if (rc) {
1561                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1562                                "Failed to find or create %s: %d\n",
1563                                libcfs_id2str(id), rc);
1564                         break;
1565                 }
1566         }
1567
1568         if (rc) {
1569                 lstcon_group_decref(tmp);
1570                 return rc;
1571         }
1572
1573         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1574
1575         lstcon_group_decref(tmp);
1576
1577         return rc;
1578 }
1579
1580 static int
1581 lstcon_debug_ndlist(struct list_head *ndlist,
1582                     struct list_head *translist,
1583                     int timeout, struct list_head __user *result_up)
1584 {
1585         struct lstcon_rpc_trans *trans;
1586         int rc;
1587
1588         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1589                                      NULL, lstcon_sesrpc_condition, &trans);
1590         if (rc) {
1591                 CERROR("Can't create transaction: %d\n", rc);
1592                 return rc;
1593         }
1594
1595         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1596
1597         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1598                                           lstcon_sesrpc_readent);
1599         lstcon_rpc_trans_destroy(trans);
1600
1601         return rc;
1602 }
1603
1604 int
1605 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1606 {
1607         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1608                                    NULL, timeout, result_up);
1609 }
1610
1611 int
1612 lstcon_batch_debug(int timeout, char *name,
1613                    int client, struct list_head __user *result_up)
1614 {
1615         struct lstcon_batch *bat;
1616         int rc;
1617
1618         rc = lstcon_batch_find(name, &bat);
1619         if (rc)
1620                 return -ENOENT;
1621
1622         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1623                                           &bat->bat_srv_list,
1624                                  NULL, timeout, result_up);
1625
1626         return rc;
1627 }
1628
1629 int
1630 lstcon_group_debug(int timeout, char *name,
1631                    struct list_head __user *result_up)
1632 {
1633         struct lstcon_group *grp;
1634         int rc;
1635
1636         rc = lstcon_group_find(name, &grp);
1637         if (rc)
1638                 return -ENOENT;
1639
1640         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1641                                  timeout, result_up);
1642         lstcon_group_decref(grp);
1643
1644         return rc;
1645 }
1646
1647 int
1648 lstcon_nodes_debug(int timeout, int count,
1649                    struct lnet_process_id __user *ids_up,
1650                    struct list_head __user *result_up)
1651 {
1652         struct lnet_process_id id;
1653         struct lstcon_ndlink *ndl;
1654         struct lstcon_group *grp;
1655         int i;
1656         int rc;
1657
1658         rc = lstcon_group_alloc(NULL, &grp);
1659         if (rc) {
1660                 CDEBUG(D_NET, "Out of memory\n");
1661                 return rc;
1662         }
1663
1664         for (i = 0; i < count; i++) {
1665                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1666                         rc = -EFAULT;
1667                         break;
1668                 }
1669
1670                 /* node is added to tmp group */
1671                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1672                 if (rc) {
1673                         CERROR("Can't create node link\n");
1674                         break;
1675                 }
1676         }
1677
1678         if (rc) {
1679                 lstcon_group_decref(grp);
1680                 return rc;
1681         }
1682
1683         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1684                                  timeout, result_up);
1685
1686         lstcon_group_decref(grp);
1687
1688         return rc;
1689 }
1690
1691 int
1692 lstcon_session_match(struct lst_sid sid)
1693 {
1694         return (console_session.ses_id.ses_nid == sid.ses_nid &&
1695                 console_session.ses_id.ses_stamp == sid.ses_stamp) ? 1 : 0;
1696 }
1697
1698 static void
1699 lstcon_new_session_id(struct lst_sid *sid)
1700 {
1701         struct lnet_process_id id;
1702
1703         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1704
1705         LNetGetId(1, &id);
1706         sid->ses_nid = id.nid;
1707         sid->ses_stamp = cfs_time_current();
1708 }
1709
1710 int
1711 lstcon_session_new(char *name, int key, unsigned int feats,
1712                    int timeout, int force, struct lst_sid __user *sid_up)
1713 {
1714         int rc = 0;
1715         int i;
1716
1717         if (console_session.ses_state != LST_SESSION_NONE) {
1718                 /* session exists */
1719                 if (!force) {
1720                         CNETERR("Session %s already exists\n",
1721                                 console_session.ses_name);
1722                         return -EEXIST;
1723                 }
1724
1725                 rc = lstcon_session_end();
1726
1727                 /* lstcon_session_end() only return local error */
1728                 if (rc)
1729                         return rc;
1730         }
1731
1732         if (feats & ~LST_FEATS_MASK) {
1733                 CNETERR("Unknown session features %x\n",
1734                         (feats & ~LST_FEATS_MASK));
1735                 return -EINVAL;
1736         }
1737
1738         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1739                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1740
1741         lstcon_new_session_id(&console_session.ses_id);
1742
1743         console_session.ses_key = key;
1744         console_session.ses_state = LST_SESSION_ACTIVE;
1745         console_session.ses_force = !!force;
1746         console_session.ses_features = feats;
1747         console_session.ses_feats_updated = 0;
1748         console_session.ses_timeout = (timeout <= 0) ?
1749                                       LST_CONSOLE_TIMEOUT : timeout;
1750
1751         if (strlen(name) > sizeof(console_session.ses_name) - 1)
1752                 return -E2BIG;
1753         strlcpy(console_session.ses_name, name,
1754                 sizeof(console_session.ses_name));
1755
1756         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1757         if (rc)
1758                 return rc;
1759
1760         rc = lstcon_rpc_pinger_start();
1761         if (rc) {
1762                 struct lstcon_batch *bat = NULL;
1763
1764                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1765                 lstcon_batch_destroy(bat);
1766
1767                 return rc;
1768         }
1769
1770         if (!copy_to_user(sid_up, &console_session.ses_id,
1771                           sizeof(struct lst_sid)))
1772                 return rc;
1773
1774         lstcon_session_end();
1775
1776         return -EFAULT;
1777 }
1778
1779 int
1780 lstcon_session_info(struct lst_sid __user *sid_up, int __user *key_up,
1781                     unsigned __user *featp,
1782                     struct lstcon_ndlist_ent __user *ndinfo_up,
1783                     char __user *name_up, int len)
1784 {
1785         struct lstcon_ndlist_ent *entp;
1786         struct lstcon_ndlink *ndl;
1787         int rc = 0;
1788
1789         if (console_session.ses_state != LST_SESSION_ACTIVE)
1790                 return -ESRCH;
1791
1792         LIBCFS_ALLOC(entp, sizeof(*entp));
1793         if (!entp)
1794                 return -ENOMEM;
1795
1796         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1797                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1798
1799         if (copy_to_user(sid_up, &console_session.ses_id,
1800                          sizeof(*sid_up)) ||
1801             copy_to_user(key_up, &console_session.ses_key,
1802                          sizeof(*key_up)) ||
1803             copy_to_user(featp, &console_session.ses_features,
1804                          sizeof(*featp)) ||
1805             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1806             copy_to_user(name_up, console_session.ses_name, len))
1807                 rc = -EFAULT;
1808
1809         LIBCFS_FREE(entp, sizeof(*entp));
1810
1811         return rc;
1812 }
1813
1814 int
1815 lstcon_session_end(void)
1816 {
1817         struct lstcon_rpc_trans *trans;
1818         struct lstcon_group *grp;
1819         struct lstcon_batch *bat;
1820         int rc = 0;
1821
1822         LASSERT(console_session.ses_state == LST_SESSION_ACTIVE);
1823
1824         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1825                                      NULL, LST_TRANS_SESEND, NULL,
1826                                      lstcon_sesrpc_condition, &trans);
1827         if (rc) {
1828                 CERROR("Can't create transaction: %d\n", rc);
1829                 return rc;
1830         }
1831
1832         console_session.ses_shutdown = 1;
1833
1834         lstcon_rpc_pinger_stop();
1835
1836         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1837
1838         lstcon_rpc_trans_destroy(trans);
1839         /* User can do nothing even rpc failed, so go on */
1840
1841         /* waiting for orphan rpcs to die */
1842         lstcon_rpc_cleanup_wait();
1843
1844         console_session.ses_id = LST_INVALID_SID;
1845         console_session.ses_state = LST_SESSION_NONE;
1846         console_session.ses_key = 0;
1847         console_session.ses_force = 0;
1848         console_session.ses_feats_updated = 0;
1849
1850         /* destroy all batches */
1851         while (!list_empty(&console_session.ses_bat_list)) {
1852                 bat = list_entry(console_session.ses_bat_list.next,
1853                                  struct lstcon_batch, bat_link);
1854
1855                 lstcon_batch_destroy(bat);
1856         }
1857
1858         /* destroy all groups */
1859         while (!list_empty(&console_session.ses_grp_list)) {
1860                 grp = list_entry(console_session.ses_grp_list.next,
1861                                  struct lstcon_group, grp_link);
1862                 LASSERT(grp->grp_ref == 1);
1863
1864                 lstcon_group_decref(grp);
1865         }
1866
1867         /* all nodes should be released */
1868         LASSERT(list_empty(&console_session.ses_ndl_list));
1869
1870         console_session.ses_shutdown = 0;
1871         console_session.ses_expired = 0;
1872
1873         return rc;
1874 }
1875
1876 int
1877 lstcon_session_feats_check(unsigned int feats)
1878 {
1879         int rc = 0;
1880
1881         if (feats & ~LST_FEATS_MASK) {
1882                 CERROR("Can't support these features: %x\n",
1883                        (feats & ~LST_FEATS_MASK));
1884                 return -EPROTO;
1885         }
1886
1887         spin_lock(&console_session.ses_rpc_lock);
1888
1889         if (!console_session.ses_feats_updated) {
1890                 console_session.ses_feats_updated = 1;
1891                 console_session.ses_features = feats;
1892         }
1893
1894         if (console_session.ses_features != feats)
1895                 rc = -EPROTO;
1896
1897         spin_unlock(&console_session.ses_rpc_lock);
1898
1899         if (rc) {
1900                 CERROR("remote features %x do not match with session features %x of console\n",
1901                        feats, console_session.ses_features);
1902         }
1903
1904         return rc;
1905 }
1906
1907 static int
1908 lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1909 {
1910         struct srpc_msg *rep    = &rpc->srpc_replymsg;
1911         struct srpc_msg *req    = &rpc->srpc_reqstbuf->buf_msg;
1912         struct srpc_join_reqst *jreq = &req->msg_body.join_reqst;
1913         struct srpc_join_reply *jrep = &rep->msg_body.join_reply;
1914         struct lstcon_group *grp = NULL;
1915         struct lstcon_ndlink *ndl;
1916         int rc = 0;
1917
1918         sfw_unpack_message(req);
1919
1920         mutex_lock(&console_session.ses_mutex);
1921
1922         jrep->join_sid = console_session.ses_id;
1923
1924         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1925                 jrep->join_status = ESRCH;
1926                 goto out;
1927         }
1928
1929         if (lstcon_session_feats_check(req->msg_ses_feats)) {
1930                 jrep->join_status = EPROTO;
1931                 goto out;
1932         }
1933
1934         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1935             !lstcon_session_match(jreq->join_sid)) {
1936                 jrep->join_status = EBUSY;
1937                 goto out;
1938         }
1939
1940         if (lstcon_group_find(jreq->join_group, &grp)) {
1941                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1942                 if (rc) {
1943                         CERROR("Out of memory\n");
1944                         goto out;
1945                 }
1946
1947                 list_add_tail(&grp->grp_link,
1948                               &console_session.ses_grp_list);
1949                 lstcon_group_addref(grp);
1950         }
1951
1952         if (grp->grp_ref > 2) {
1953                 /* Group in using */
1954                 jrep->join_status = EBUSY;
1955                 goto out;
1956         }
1957
1958         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1959         if (!rc) {
1960                 jrep->join_status = EEXIST;
1961                 goto out;
1962         }
1963
1964         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1965         if (rc) {
1966                 CERROR("Out of memory\n");
1967                 goto out;
1968         }
1969
1970         ndl->ndl_node->nd_state = LST_NODE_ACTIVE;
1971         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1972
1973         if (!grp->grp_userland)
1974                 grp->grp_userland = 1;
1975
1976         strlcpy(jrep->join_session, console_session.ses_name,
1977                 sizeof(jrep->join_session));
1978         jrep->join_timeout = console_session.ses_timeout;
1979         jrep->join_status = 0;
1980
1981 out:
1982         rep->msg_ses_feats = console_session.ses_features;
1983         if (grp)
1984                 lstcon_group_decref(grp);
1985
1986         mutex_unlock(&console_session.ses_mutex);
1987
1988         return rc;
1989 }
1990
1991 static struct srpc_service lstcon_acceptor_service;
1992
1993 static void lstcon_init_acceptor_service(void)
1994 {
1995         /* initialize selftest console acceptor service table */
1996         lstcon_acceptor_service.sv_name = "join session";
1997         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
1998         lstcon_acceptor_service.sv_id = SRPC_SERVICE_JOIN;
1999         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
2000 }
2001
2002 static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
2003
2004 /* initialize console */
2005 int
2006 lstcon_console_init(void)
2007 {
2008         int i;
2009         int rc;
2010
2011         memset(&console_session, 0, sizeof(struct lstcon_session));
2012
2013         console_session.ses_id = LST_INVALID_SID;
2014         console_session.ses_state = LST_SESSION_NONE;
2015         console_session.ses_timeout = 0;
2016         console_session.ses_force = 0;
2017         console_session.ses_expired = 0;
2018         console_session.ses_feats_updated = 0;
2019         console_session.ses_features = LST_FEATS_MASK;
2020         console_session.ses_laststamp = ktime_get_real_seconds();
2021
2022         mutex_init(&console_session.ses_mutex);
2023
2024         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2025         INIT_LIST_HEAD(&console_session.ses_grp_list);
2026         INIT_LIST_HEAD(&console_session.ses_bat_list);
2027         INIT_LIST_HEAD(&console_session.ses_trans_list);
2028
2029         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2030                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2031         if (!console_session.ses_ndl_hash)
2032                 return -ENOMEM;
2033
2034         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2035                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2036
2037         /* initialize acceptor service table */
2038         lstcon_init_acceptor_service();
2039
2040         rc = srpc_add_service(&lstcon_acceptor_service);
2041         LASSERT(rc != -EBUSY);
2042         if (rc) {
2043                 LIBCFS_FREE(console_session.ses_ndl_hash,
2044                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2045                 return rc;
2046         }
2047
2048         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2049                                       lstcon_acceptor_service.sv_wi_total);
2050         if (rc) {
2051                 rc = -ENOMEM;
2052                 goto out;
2053         }
2054
2055         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2056
2057         if (!rc) {
2058                 lstcon_rpc_module_init();
2059                 return 0;
2060         }
2061
2062 out:
2063         srpc_shutdown_service(&lstcon_acceptor_service);
2064         srpc_remove_service(&lstcon_acceptor_service);
2065
2066         LIBCFS_FREE(console_session.ses_ndl_hash,
2067                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2068
2069         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2070
2071         return rc;
2072 }
2073
2074 int
2075 lstcon_console_fini(void)
2076 {
2077         int i;
2078
2079         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2080
2081         mutex_lock(&console_session.ses_mutex);
2082
2083         srpc_shutdown_service(&lstcon_acceptor_service);
2084         srpc_remove_service(&lstcon_acceptor_service);
2085
2086         if (console_session.ses_state != LST_SESSION_NONE)
2087                 lstcon_session_end();
2088
2089         lstcon_rpc_module_fini();
2090
2091         mutex_unlock(&console_session.ses_mutex);
2092
2093         LASSERT(list_empty(&console_session.ses_ndl_list));
2094         LASSERT(list_empty(&console_session.ses_grp_list));
2095         LASSERT(list_empty(&console_session.ses_bat_list));
2096         LASSERT(list_empty(&console_session.ses_trans_list));
2097
2098         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2099                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2100
2101         LIBCFS_FREE(console_session.ses_ndl_hash,
2102                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2103
2104         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2105
2106         return 0;
2107 }