drivers/net/team/team_mode_loadbalance.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team
   4  * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com>
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/types.h>
   9 #include <linux/module.h>
  10 #include <linux/init.h>
  11 #include <linux/errno.h>
  12 #include <linux/netdevice.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/filter.h>
  15 #include <linux/if_team.h>
  16
  17 static rx_handler_result_t lb_receive(struct team *team, struct team_port *port,
  18                                       struct sk_buff *skb)
  19 {
  20         if (unlikely(skb->protocol == htons(ETH_P_SLOW))) {
  21                 /* LACPDU packets should go to exact delivery */
  22                 const unsigned char *dest = eth_hdr(skb)->h_dest;
  23
  24                 if (is_link_local_ether_addr(dest) && dest[5] == 0x02)
  25                         return RX_HANDLER_EXACT;
  26         }
  27         return RX_HANDLER_ANOTHER;
  28 }
  29
  30 struct lb_priv;
  31
  32 typedef struct team_port *lb_select_tx_port_func_t(struct team *,
  33                                                    struct lb_priv *,
  34                                                    struct sk_buff *,
  35                                                    unsigned char);
  36
  37 #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */
  38
  39 struct lb_stats {
  40         u64 tx_bytes;
  41 };
  42
  43 struct lb_pcpu_stats {
  44         struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE];
  45         struct u64_stats_sync syncp;
  46 };
  47
  48 struct lb_stats_info {
  49         struct lb_stats stats;
  50         struct lb_stats last_stats;
  51         struct team_option_inst_info *opt_inst_info;
  52 };
  53
  54 struct lb_port_mapping {
  55         struct team_port __rcu *port;
  56         struct team_option_inst_info *opt_inst_info;
  57 };
  58
  59 struct lb_priv_ex {
  60         struct team *team;
  61         struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE];
  62         struct sock_fprog_kern *orig_fprog;
  63         struct {
  64                 unsigned int refresh_interval; /* in tenths of second */
  65                 struct delayed_work refresh_dw;
  66                 struct lb_stats_info info[LB_TX_HASHTABLE_SIZE];
  67         } stats;
  68 };
  69
  70 struct lb_priv {
  71         struct bpf_prog __rcu *fp;
  72         lb_select_tx_port_func_t __rcu *select_tx_port_func;
  73         struct lb_pcpu_stats __percpu *pcpu_stats;
  74         struct lb_priv_ex *ex; /* priv extension */
  75 };
  76
  77 static struct lb_priv *get_lb_priv(struct team *team)
  78 {
  79         return (struct lb_priv *) &team->mode_priv;
  80 }
  81
  82 struct lb_port_priv {
  83         struct lb_stats __percpu *pcpu_stats;
  84         struct lb_stats_info stats_info;
  85 };
  86
  87 static struct lb_port_priv *get_lb_port_priv(struct team_port *port)
  88 {
  89         return (struct lb_port_priv *) &port->mode_priv;
  90 }
  91
  92 #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \
  93         (lb_priv)->ex->tx_hash_to_port_mapping[hash].port
  94
  95 #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \
  96         (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info
  97
  98 static void lb_tx_hash_to_port_mapping_null_port(struct team *team,
  99                                                  struct team_port *port)
 100 {
 101         struct lb_priv *lb_priv = get_lb_priv(team);
 102         bool changed = false;
 103         int i;
 104
 105         for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) {
 106                 struct lb_port_mapping *pm;
 107
 108                 pm = &lb_priv->ex->tx_hash_to_port_mapping[i];
 109                 if (rcu_access_pointer(pm->port) == port) {
 110                         RCU_INIT_POINTER(pm->port, NULL);
 111                         team_option_inst_set_change(pm->opt_inst_info);
 112                         changed = true;
 113                 }
 114         }
 115         if (changed)
 116                 team_options_change_check(team);
 117 }
 118
 119 /* Basic tx selection based solely by hash */
 120 static struct team_port *lb_hash_select_tx_port(struct team *team,
 121                                                 struct lb_priv *lb_priv,
 122                                                 struct sk_buff *skb,
 123                                                 unsigned char hash)
 124 {
 125         int port_index = team_num_to_port_index(team, hash);
 126
 127         return team_get_port_by_index_rcu(team, port_index);
 128 }
 129
 130 /* Hash to port mapping select tx port */
 131 static struct team_port *lb_htpm_select_tx_port(struct team *team,
 132                                                 struct lb_priv *lb_priv,
 133                                                 struct sk_buff *skb,
 134                                                 unsigned char hash)
 135 {
 136         struct team_port *port;
 137
 138         port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash));
 139         if (likely(port))
 140                 return port;
 141         /* If no valid port in the table, fall back to simple hash */
 142         return lb_hash_select_tx_port(team, lb_priv, skb, hash);
 143 }
 144
 145 struct lb_select_tx_port {
 146         char *name;
 147         lb_select_tx_port_func_t *func;
 148 };
 149
 150 static const struct lb_select_tx_port lb_select_tx_port_list[] = {
 151         {
 152                 .name = "hash",
 153                 .func = lb_hash_select_tx_port,
 154         },
 155         {
 156                 .name = "hash_to_port_mapping",
 157                 .func = lb_htpm_select_tx_port,
 158         },
 159 };
 160 #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list)
 161
 162 static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func)
 163 {
 164         int i;
 165
 166         for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
 167                 const struct lb_select_tx_port *item;
 168
 169                 item = &lb_select_tx_port_list[i];
 170                 if (item->func == func)
 171                         return item->name;
 172         }
 173         return NULL;
 174 }
 175
 176 static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name)
 177 {
 178         int i;
 179
 180         for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
 181                 const struct lb_select_tx_port *item;
 182
 183                 item = &lb_select_tx_port_list[i];
 184                 if (!strcmp(item->name, name))
 185                         return item->func;
 186         }
 187         return NULL;
 188 }
 189
 190 static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv,
 191                                     struct sk_buff *skb)
 192 {
 193         struct bpf_prog *fp;
 194         uint32_t lhash;
 195         unsigned char *c;
 196
 197         fp = rcu_dereference_bh(lb_priv->fp);
 198         if (unlikely(!fp))
 199                 return 0;
 200         lhash = BPF_PROG_RUN(fp, skb);
 201         c = (char *) &lhash;
 202         return c[0] ^ c[1] ^ c[2] ^ c[3];
 203 }
 204
 205 static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv,
 206                                struct lb_port_priv *lb_port_priv,
 207                                unsigned char hash)
 208 {
 209         struct lb_pcpu_stats *pcpu_stats;
 210         struct lb_stats *port_stats;
 211         struct lb_stats *hash_stats;
 212
 213         pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats);
 214         port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats);
 215         hash_stats = &pcpu_stats->hash_stats[hash];
 216         u64_stats_update_begin(&pcpu_stats->syncp);
 217         port_stats->tx_bytes += tx_bytes;
 218         hash_stats->tx_bytes += tx_bytes;
 219         u64_stats_update_end(&pcpu_stats->syncp);
 220 }
 221
 222 static bool lb_transmit(struct team *team, struct sk_buff *skb)
 223 {
 224         struct lb_priv *lb_priv = get_lb_priv(team);
 225         lb_select_tx_port_func_t *select_tx_port_func;
 226         struct team_port *port;
 227         unsigned char hash;
 228         unsigned int tx_bytes = skb->len;
 229
 230         hash = lb_get_skb_hash(lb_priv, skb);
 231         select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func);
 232         port = select_tx_port_func(team, lb_priv, skb, hash);
 233         if (unlikely(!port))
 234                 goto drop;
 235         if (team_dev_queue_xmit(team, port, skb))
 236                 return false;
 237         lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash);
 238         return true;
 239
 240 drop:
 241         dev_kfree_skb_any(skb);
 242         return false;
 243 }
 244
 245 static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx)
 246 {
 247         struct lb_priv *lb_priv = get_lb_priv(team);
 248
 249         if (!lb_priv->ex->orig_fprog) {
 250                 ctx->data.bin_val.len = 0;
 251                 ctx->data.bin_val.ptr = NULL;
 252                 return 0;
 253         }
 254         ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len *
 255                                 sizeof(struct sock_filter);
 256         ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter;
 257         return 0;
 258 }
 259
 260 static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len,
 261                           const void *data)
 262 {
 263         struct sock_fprog_kern *fprog;
 264         struct sock_filter *filter = (struct sock_filter *) data;
 265
 266         if (data_len % sizeof(struct sock_filter))
 267                 return -EINVAL;
 268         fprog = kmalloc(sizeof(*fprog), GFP_KERNEL);
 269         if (!fprog)
 270                 return -ENOMEM;
 271         fprog->filter = kmemdup(filter, data_len, GFP_KERNEL);
 272         if (!fprog->filter) {
 273                 kfree(fprog);
 274                 return -ENOMEM;
 275         }
 276         fprog->len = data_len / sizeof(struct sock_filter);
 277         *pfprog = fprog;
 278         return 0;
 279 }
 280
 281 static void __fprog_destroy(struct sock_fprog_kern *fprog)
 282 {
 283         kfree(fprog->filter);
 284         kfree(fprog);
 285 }
 286
 287 static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx)
 288 {
 289         struct lb_priv *lb_priv = get_lb_priv(team);
 290         struct bpf_prog *fp = NULL;
 291         struct bpf_prog *orig_fp = NULL;
 292         struct sock_fprog_kern *fprog = NULL;
 293         int err;
 294
 295         if (ctx->data.bin_val.len) {
 296                 err = __fprog_create(&fprog, ctx->data.bin_val.len,
 297                                      ctx->data.bin_val.ptr);
 298                 if (err)
 299                         return err;
 300                 err = bpf_prog_create(&fp, fprog);
 301                 if (err) {
 302                         __fprog_destroy(fprog);
 303                         return err;
 304                 }
 305         }
 306
 307         if (lb_priv->ex->orig_fprog) {
 308                 /* Clear old filter data */
 309                 __fprog_destroy(lb_priv->ex->orig_fprog);
 310                 orig_fp = rcu_dereference_protected(lb_priv->fp,
 311                                                 lockdep_is_held(&team->lock));
 312         }
 313
 314         rcu_assign_pointer(lb_priv->fp, fp);
 315         lb_priv->ex->orig_fprog = fprog;
 316
 317         if (orig_fp) {
 318                 synchronize_rcu();
 319                 bpf_prog_destroy(orig_fp);
 320         }
 321         return 0;
 322 }
 323
 324 static void lb_bpf_func_free(struct team *team)
 325 {
 326         struct lb_priv *lb_priv = get_lb_priv(team);
 327         struct bpf_prog *fp;
 328
 329         if (!lb_priv->ex->orig_fprog)
 330                 return;
 331
 332         __fprog_destroy(lb_priv->ex->orig_fprog);
 333         fp = rcu_dereference_protected(lb_priv->fp,
 334                                        lockdep_is_held(&team->lock));
 335         bpf_prog_destroy(fp);
 336 }
 337
 338 static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx)
 339 {
 340         struct lb_priv *lb_priv = get_lb_priv(team);
 341         lb_select_tx_port_func_t *func;
 342         char *name;
 343
 344         func = rcu_dereference_protected(lb_priv->select_tx_port_func,
 345                                          lockdep_is_held(&team->lock));
 346         name = lb_select_tx_port_get_name(func);
 347         BUG_ON(!name);
 348         ctx->data.str_val = name;
 349         return 0;
 350 }
 351
 352 static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx)
 353 {
 354         struct lb_priv *lb_priv = get_lb_priv(team);
 355         lb_select_tx_port_func_t *func;
 356
 357         func = lb_select_tx_port_get_func(ctx->data.str_val);
 358         if (!func)
 359                 return -EINVAL;
 360         rcu_assign_pointer(lb_priv->select_tx_port_func, func);
 361         return 0;
 362 }
 363
 364 static int lb_tx_hash_to_port_mapping_init(struct team *team,
 365                                            struct team_option_inst_info *info)
 366 {
 367         struct lb_priv *lb_priv = get_lb_priv(team);
 368         unsigned char hash = info->array_index;
 369
 370         LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info;
 371         return 0;
 372 }
 373
 374 static int lb_tx_hash_to_port_mapping_get(struct team *team,
 375                                           struct team_gsetter_ctx *ctx)
 376 {
 377         struct lb_priv *lb_priv = get_lb_priv(team);
 378         struct team_port *port;
 379         unsigned char hash = ctx->info->array_index;
 380
 381         port = LB_HTPM_PORT_BY_HASH(lb_priv, hash);
 382         ctx->data.u32_val = port ? port->dev->ifindex : 0;
 383         return 0;
 384 }
 385
 386 static int lb_tx_hash_to_port_mapping_set(struct team *team,
 387                                           struct team_gsetter_ctx *ctx)
 388 {
 389         struct lb_priv *lb_priv = get_lb_priv(team);
 390         struct team_port *port;
 391         unsigned char hash = ctx->info->array_index;
 392
 393         list_for_each_entry(port, &team->port_list, list) {
 394                 if (ctx->data.u32_val == port->dev->ifindex &&
 395                     team_port_enabled(port)) {
 396                         rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash),
 397                                            port);
 398                         return 0;
 399                 }
 400         }
 401         return -ENODEV;
 402 }
 403
 404 static int lb_hash_stats_init(struct team *team,
 405                               struct team_option_inst_info *info)
 406 {
 407         struct lb_priv *lb_priv = get_lb_priv(team);
 408         unsigned char hash = info->array_index;
 409
 410         lb_priv->ex->stats.info[hash].opt_inst_info = info;
 411         return 0;
 412 }
 413
 414 static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
 415 {
 416         struct lb_priv *lb_priv = get_lb_priv(team);
 417         unsigned char hash = ctx->info->array_index;
 418
 419         ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats;
 420         ctx->data.bin_val.len = sizeof(struct lb_stats);
 421         return 0;
 422 }
 423
 424 static int lb_port_stats_init(struct team *team,
 425                               struct team_option_inst_info *info)
 426 {
 427         struct team_port *port = info->port;
 428         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 429
 430         lb_port_priv->stats_info.opt_inst_info = info;
 431         return 0;
 432 }
 433
 434 static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
 435 {
 436         struct team_port *port = ctx->info->port;
 437         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 438
 439         ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats;
 440         ctx->data.bin_val.len = sizeof(struct lb_stats);
 441         return 0;
 442 }
 443
 444 static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info)
 445 {
 446         memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats));
 447         memset(&s_info->stats, 0, sizeof(struct lb_stats));
 448 }
 449
 450 static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info,
 451                                           struct team *team)
 452 {
 453         if (memcmp(&s_info->last_stats, &s_info->stats,
 454             sizeof(struct lb_stats))) {
 455                 team_option_inst_set_change(s_info->opt_inst_info);
 456                 return true;
 457         }
 458         return false;
 459 }
 460
 461 static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats,
 462                                    struct lb_stats *cpu_stats,
 463                                    struct u64_stats_sync *syncp)
 464 {
 465         unsigned int start;
 466         struct lb_stats tmp;
 467
 468         do {
 469                 start = u64_stats_fetch_begin_irq(syncp);
 470                 tmp.tx_bytes = cpu_stats->tx_bytes;
 471         } while (u64_stats_fetch_retry_irq(syncp, start));
 472         acc_stats->tx_bytes += tmp.tx_bytes;
 473 }
 474
 475 static void lb_stats_refresh(struct work_struct *work)
 476 {
 477         struct team *team;
 478         struct lb_priv *lb_priv;
 479         struct lb_priv_ex *lb_priv_ex;
 480         struct lb_pcpu_stats *pcpu_stats;
 481         struct lb_stats *stats;
 482         struct lb_stats_info *s_info;
 483         struct team_port *port;
 484         bool changed = false;
 485         int i;
 486         int j;
 487
 488         lb_priv_ex = container_of(work, struct lb_priv_ex,
 489                                   stats.refresh_dw.work);
 490
 491         team = lb_priv_ex->team;
 492         lb_priv = get_lb_priv(team);
 493
 494         if (!mutex_trylock(&team->lock)) {
 495                 schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0);
 496                 return;
 497         }
 498
 499         for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) {
 500                 s_info = &lb_priv->ex->stats.info[j];
 501                 __lb_stats_info_refresh_prepare(s_info);
 502                 for_each_possible_cpu(i) {
 503                         pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 504                         stats = &pcpu_stats->hash_stats[j];
 505                         __lb_one_cpu_stats_add(&s_info->stats, stats,
 506                                                &pcpu_stats->syncp);
 507                 }
 508                 changed |= __lb_stats_info_refresh_check(s_info, team);
 509         }
 510
 511         list_for_each_entry(port, &team->port_list, list) {
 512                 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 513
 514                 s_info = &lb_port_priv->stats_info;
 515                 __lb_stats_info_refresh_prepare(s_info);
 516                 for_each_possible_cpu(i) {
 517                         pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 518                         stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i);
 519                         __lb_one_cpu_stats_add(&s_info->stats, stats,
 520                                                &pcpu_stats->syncp);
 521                 }
 522                 changed |= __lb_stats_info_refresh_check(s_info, team);
 523         }
 524
 525         if (changed)
 526                 team_options_change_check(team);
 527
 528         schedule_delayed_work(&lb_priv_ex->stats.refresh_dw,
 529                               (lb_priv_ex->stats.refresh_interval * HZ) / 10);
 530
 531         mutex_unlock(&team->lock);
 532 }
 533
 534 static int lb_stats_refresh_interval_get(struct team *team,
 535                                          struct team_gsetter_ctx *ctx)
 536 {
 537         struct lb_priv *lb_priv = get_lb_priv(team);
 538
 539         ctx->data.u32_val = lb_priv->ex->stats.refresh_interval;
 540         return 0;
 541 }
 542
 543 static int lb_stats_refresh_interval_set(struct team *team,
 544                                          struct team_gsetter_ctx *ctx)
 545 {
 546         struct lb_priv *lb_priv = get_lb_priv(team);
 547         unsigned int interval;
 548
 549         interval = ctx->data.u32_val;
 550         if (lb_priv->ex->stats.refresh_interval == interval)
 551                 return 0;
 552         lb_priv->ex->stats.refresh_interval = interval;
 553         if (interval)
 554                 schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0);
 555         else
 556                 cancel_delayed_work(&lb_priv->ex->stats.refresh_dw);
 557         return 0;
 558 }
 559
 560 static const struct team_option lb_options[] = {
 561         {
 562                 .name = "bpf_hash_func",
 563                 .type = TEAM_OPTION_TYPE_BINARY,
 564                 .getter = lb_bpf_func_get,
 565                 .setter = lb_bpf_func_set,
 566         },
 567         {
 568                 .name = "lb_tx_method",
 569                 .type = TEAM_OPTION_TYPE_STRING,
 570                 .getter = lb_tx_method_get,
 571                 .setter = lb_tx_method_set,
 572         },
 573         {
 574                 .name = "lb_tx_hash_to_port_mapping",
 575                 .array_size = LB_TX_HASHTABLE_SIZE,
 576                 .type = TEAM_OPTION_TYPE_U32,
 577                 .init = lb_tx_hash_to_port_mapping_init,
 578                 .getter = lb_tx_hash_to_port_mapping_get,
 579                 .setter = lb_tx_hash_to_port_mapping_set,
 580         },
 581         {
 582                 .name = "lb_hash_stats",
 583                 .array_size = LB_TX_HASHTABLE_SIZE,
 584                 .type = TEAM_OPTION_TYPE_BINARY,
 585                 .init = lb_hash_stats_init,
 586                 .getter = lb_hash_stats_get,
 587         },
 588         {
 589                 .name = "lb_port_stats",
 590                 .per_port = true,
 591                 .type = TEAM_OPTION_TYPE_BINARY,
 592                 .init = lb_port_stats_init,
 593                 .getter = lb_port_stats_get,
 594         },
 595         {
 596                 .name = "lb_stats_refresh_interval",
 597                 .type = TEAM_OPTION_TYPE_U32,
 598                 .getter = lb_stats_refresh_interval_get,
 599                 .setter = lb_stats_refresh_interval_set,
 600         },
 601 };
 602
 603 static int lb_init(struct team *team)
 604 {
 605         struct lb_priv *lb_priv = get_lb_priv(team);
 606         lb_select_tx_port_func_t *func;
 607         int i, err;
 608
 609         /* set default tx port selector */
 610         func = lb_select_tx_port_get_func("hash");
 611         BUG_ON(!func);
 612         rcu_assign_pointer(lb_priv->select_tx_port_func, func);
 613
 614         lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL);
 615         if (!lb_priv->ex)
 616                 return -ENOMEM;
 617         lb_priv->ex->team = team;
 618
 619         lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats);
 620         if (!lb_priv->pcpu_stats) {
 621                 err = -ENOMEM;
 622                 goto err_alloc_pcpu_stats;
 623         }
 624
 625         for_each_possible_cpu(i) {
 626                 struct lb_pcpu_stats *team_lb_stats;
 627                 team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 628                 u64_stats_init(&team_lb_stats->syncp);
 629         }
 630
 631
 632         INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh);
 633
 634         err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options));
 635         if (err)
 636                 goto err_options_register;
 637         return 0;
 638
 639 err_options_register:
 640         free_percpu(lb_priv->pcpu_stats);
 641 err_alloc_pcpu_stats:
 642         kfree(lb_priv->ex);
 643         return err;
 644 }
 645
 646 static void lb_exit(struct team *team)
 647 {
 648         struct lb_priv *lb_priv = get_lb_priv(team);
 649
 650         team_options_unregister(team, lb_options,
 651                                 ARRAY_SIZE(lb_options));
 652         lb_bpf_func_free(team);
 653         cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw);
 654         free_percpu(lb_priv->pcpu_stats);
 655         kfree(lb_priv->ex);
 656 }
 657
 658 static int lb_port_enter(struct team *team, struct team_port *port)
 659 {
 660         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 661
 662         lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats);
 663         if (!lb_port_priv->pcpu_stats)
 664                 return -ENOMEM;
 665         return 0;
 666 }
 667
 668 static void lb_port_leave(struct team *team, struct team_port *port)
 669 {
 670         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 671
 672         free_percpu(lb_port_priv->pcpu_stats);
 673 }
 674
 675 static void lb_port_disabled(struct team *team, struct team_port *port)
 676 {
 677         lb_tx_hash_to_port_mapping_null_port(team, port);
 678 }
 679
 680 static const struct team_mode_ops lb_mode_ops = {
 681         .init                   = lb_init,
 682         .exit                   = lb_exit,
 683         .port_enter             = lb_port_enter,
 684         .port_leave             = lb_port_leave,
 685         .port_disabled          = lb_port_disabled,
 686         .receive                = lb_receive,
 687         .transmit               = lb_transmit,
 688 };
 689
 690 static const struct team_mode lb_mode = {
 691         .kind           = "loadbalance",
 692         .owner          = THIS_MODULE,
 693         .priv_size      = sizeof(struct lb_priv),
 694         .port_priv_size = sizeof(struct lb_port_priv),
 695         .ops            = &lb_mode_ops,
 696         .lag_tx_type    = NETDEV_LAG_TX_TYPE_HASH,
 697 };
 698
 699 static int __init lb_init_module(void)
 700 {
 701         return team_mode_register(&lb_mode);
 702 }
 703
 704 static void __exit lb_cleanup_module(void)
 705 {
 706         team_mode_unregister(&lb_mode);
 707 }
 708
 709 module_init(lb_init_module);
 710 module_exit(lb_cleanup_module);
 711
 712 MODULE_LICENSE("GPL v2");
 713 MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>");
 714 MODULE_DESCRIPTION("Load-balancing mode for team");
 715 MODULE_ALIAS_TEAM_MODE("loadbalance");