drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/slab.h>
   6 #include <linux/list.h>
   7 #include <linux/errno.h>
   8
   9 #include "item.h"
  10 #include "core_acl_flex_keys.h"
  11
  12 struct mlxsw_afk {
  13         struct list_head key_info_list;
  14         unsigned int max_blocks;
  15         const struct mlxsw_afk_ops *ops;
  16         const struct mlxsw_afk_block *blocks;
  17         unsigned int blocks_count;
  18 };
  19
  20 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk)
  21 {
  22         int i;
  23         int j;
  24
  25         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
  26                 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
  27
  28                 for (j = 0; j < block->instances_count; j++) {
  29                         struct mlxsw_afk_element_inst *elinst;
  30
  31                         elinst = &block->instances[j];
  32                         if (elinst->type != elinst->info->type ||
  33                             (!elinst->avoid_size_check &&
  34                              elinst->item.size.bits !=
  35                              elinst->info->item.size.bits))
  36                                 return false;
  37                 }
  38         }
  39         return true;
  40 }
  41
  42 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks,
  43                                    const struct mlxsw_afk_ops *ops)
  44 {
  45         struct mlxsw_afk *mlxsw_afk;
  46
  47         mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL);
  48         if (!mlxsw_afk)
  49                 return NULL;
  50         INIT_LIST_HEAD(&mlxsw_afk->key_info_list);
  51         mlxsw_afk->max_blocks = max_blocks;
  52         mlxsw_afk->ops = ops;
  53         mlxsw_afk->blocks = ops->blocks;
  54         mlxsw_afk->blocks_count = ops->blocks_count;
  55         WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk));
  56         return mlxsw_afk;
  57 }
  58 EXPORT_SYMBOL(mlxsw_afk_create);
  59
  60 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk)
  61 {
  62         WARN_ON(!list_empty(&mlxsw_afk->key_info_list));
  63         kfree(mlxsw_afk);
  64 }
  65 EXPORT_SYMBOL(mlxsw_afk_destroy);
  66
  67 struct mlxsw_afk_key_info {
  68         struct list_head list;
  69         unsigned int ref_count;
  70         unsigned int blocks_count;
  71         int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value
  72                                                       * is index inside "blocks"
  73                                                       */
  74         struct mlxsw_afk_element_usage elusage;
  75         const struct mlxsw_afk_block *blocks[0];
  76 };
  77
  78 static bool
  79 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info,
  80                                struct mlxsw_afk_element_usage *elusage)
  81 {
  82         return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0;
  83 }
  84
  85 static struct mlxsw_afk_key_info *
  86 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk,
  87                         struct mlxsw_afk_element_usage *elusage)
  88 {
  89         struct mlxsw_afk_key_info *key_info;
  90
  91         list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) {
  92                 if (mlxsw_afk_key_info_elements_eq(key_info, elusage))
  93                         return key_info;
  94         }
  95         return NULL;
  96 }
  97
  98 struct mlxsw_afk_picker {
  99         struct {
 100                 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX);
 101                 unsigned int total;
 102         } hits[0];
 103 };
 104
 105 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk,
 106                                         struct mlxsw_afk_picker *picker,
 107                                         enum mlxsw_afk_element element)
 108 {
 109         int i;
 110         int j;
 111
 112         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 113                 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
 114
 115                 for (j = 0; j < block->instances_count; j++) {
 116                         struct mlxsw_afk_element_inst *elinst;
 117
 118                         elinst = &block->instances[j];
 119                         if (elinst->info->element == element) {
 120                                 __set_bit(element, picker->hits[i].element);
 121                                 picker->hits[i].total++;
 122                         }
 123                 }
 124         }
 125 }
 126
 127 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk,
 128                                            struct mlxsw_afk_picker *picker,
 129                                            int block_index)
 130 {
 131         DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX);
 132         int i;
 133         int j;
 134
 135         memcpy(&hits_element, &picker->hits[block_index].element,
 136                sizeof(hits_element));
 137
 138         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 139                 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) {
 140                         if (__test_and_clear_bit(j, picker->hits[i].element))
 141                                 picker->hits[i].total--;
 142                 }
 143         }
 144 }
 145
 146 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk,
 147                                           struct mlxsw_afk_picker *picker)
 148 {
 149         int most_index = -EINVAL; /* Should never happen to return this */
 150         int most_hits = 0;
 151         int i;
 152
 153         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 154                 if (picker->hits[i].total > most_hits) {
 155                         most_hits = picker->hits[i].total;
 156                         most_index = i;
 157                 }
 158         }
 159         return most_index;
 160 }
 161
 162 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk,
 163                                          struct mlxsw_afk_picker *picker,
 164                                          int block_index,
 165                                          struct mlxsw_afk_key_info *key_info)
 166 {
 167         enum mlxsw_afk_element element;
 168
 169         if (key_info->blocks_count == mlxsw_afk->max_blocks)
 170                 return -EINVAL;
 171
 172         for_each_set_bit(element, picker->hits[block_index].element,
 173                          MLXSW_AFK_ELEMENT_MAX) {
 174                 key_info->element_to_block[element] = key_info->blocks_count;
 175                 mlxsw_afk_element_usage_add(&key_info->elusage, element);
 176         }
 177
 178         key_info->blocks[key_info->blocks_count] =
 179                                         &mlxsw_afk->blocks[block_index];
 180         key_info->blocks_count++;
 181         return 0;
 182 }
 183
 184 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk,
 185                             struct mlxsw_afk_key_info *key_info,
 186                             struct mlxsw_afk_element_usage *elusage)
 187 {
 188         struct mlxsw_afk_picker *picker;
 189         enum mlxsw_afk_element element;
 190         size_t alloc_size;
 191         int err;
 192
 193         alloc_size = sizeof(picker->hits[0]) * mlxsw_afk->blocks_count;
 194         picker = kzalloc(alloc_size, GFP_KERNEL);
 195         if (!picker)
 196                 return -ENOMEM;
 197
 198         /* Since the same elements could be present in multiple blocks,
 199          * we must find out optimal block list in order to make the
 200          * block count as low as possible.
 201          *
 202          * First, we count hits. We go over all available blocks and count
 203          * how many of requested elements are covered by each.
 204          *
 205          * Then in loop, we find block with most hits and add it to
 206          * output key_info. Then we have to subtract this block hits so
 207          * the next iteration will find most suitable block for
 208          * the rest of requested elements.
 209          */
 210
 211         mlxsw_afk_element_usage_for_each(element, elusage)
 212                 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element);
 213
 214         do {
 215                 int block_index;
 216
 217                 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker);
 218                 if (block_index < 0) {
 219                         err = block_index;
 220                         goto out;
 221                 }
 222                 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker,
 223                                                     block_index, key_info);
 224                 if (err)
 225                         goto out;
 226                 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index);
 227         } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage));
 228
 229         err = 0;
 230 out:
 231         kfree(picker);
 232         return err;
 233 }
 234
 235 static struct mlxsw_afk_key_info *
 236 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk,
 237                           struct mlxsw_afk_element_usage *elusage)
 238 {
 239         struct mlxsw_afk_key_info *key_info;
 240         int err;
 241
 242         key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks),
 243                            GFP_KERNEL);
 244         if (!key_info)
 245                 return ERR_PTR(-ENOMEM);
 246         err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage);
 247         if (err)
 248                 goto err_picker;
 249         list_add(&key_info->list, &mlxsw_afk->key_info_list);
 250         key_info->ref_count = 1;
 251         return key_info;
 252
 253 err_picker:
 254         kfree(key_info);
 255         return ERR_PTR(err);
 256 }
 257
 258 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info)
 259 {
 260         list_del(&key_info->list);
 261         kfree(key_info);
 262 }
 263
 264 struct mlxsw_afk_key_info *
 265 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk,
 266                        struct mlxsw_afk_element_usage *elusage)
 267 {
 268         struct mlxsw_afk_key_info *key_info;
 269
 270         key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage);
 271         if (key_info) {
 272                 key_info->ref_count++;
 273                 return key_info;
 274         }
 275         return mlxsw_afk_key_info_create(mlxsw_afk, elusage);
 276 }
 277 EXPORT_SYMBOL(mlxsw_afk_key_info_get);
 278
 279 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info)
 280 {
 281         if (--key_info->ref_count)
 282                 return;
 283         mlxsw_afk_key_info_destroy(key_info);
 284 }
 285 EXPORT_SYMBOL(mlxsw_afk_key_info_put);
 286
 287 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info,
 288                                struct mlxsw_afk_element_usage *elusage)
 289 {
 290         return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage);
 291 }
 292 EXPORT_SYMBOL(mlxsw_afk_key_info_subset);
 293
 294 static const struct mlxsw_afk_element_inst *
 295 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block,
 296                            enum mlxsw_afk_element element)
 297 {
 298         int i;
 299
 300         for (i = 0; i < block->instances_count; i++) {
 301                 struct mlxsw_afk_element_inst *elinst;
 302
 303                 elinst = &block->instances[i];
 304                 if (elinst->info->element == element)
 305                         return elinst;
 306         }
 307         return NULL;
 308 }
 309
 310 static const struct mlxsw_afk_element_inst *
 311 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info,
 312                               enum mlxsw_afk_element element,
 313                               int *p_block_index)
 314 {
 315         const struct mlxsw_afk_element_inst *elinst;
 316         const struct mlxsw_afk_block *block;
 317         int block_index;
 318
 319         if (WARN_ON(!test_bit(element, key_info->elusage.usage)))
 320                 return NULL;
 321         block_index = key_info->element_to_block[element];
 322         block = key_info->blocks[block_index];
 323
 324         elinst = mlxsw_afk_block_elinst_get(block, element);
 325         if (WARN_ON(!elinst))
 326                 return NULL;
 327
 328         *p_block_index = block_index;
 329         return elinst;
 330 }
 331
 332 u16
 333 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info,
 334                                       int block_index)
 335 {
 336         return key_info->blocks[block_index]->encoding;
 337 }
 338 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get);
 339
 340 unsigned int
 341 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info)
 342 {
 343         return key_info->blocks_count;
 344 }
 345 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get);
 346
 347 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values,
 348                               enum mlxsw_afk_element element,
 349                               u32 key_value, u32 mask_value)
 350 {
 351         const struct mlxsw_afk_element_info *elinfo =
 352                                 &mlxsw_afk_element_infos[element];
 353         const struct mlxsw_item *storage_item = &elinfo->item;
 354
 355         if (!mask_value)
 356                 return;
 357         if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32))
 358                 return;
 359         __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value);
 360         __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value);
 361         mlxsw_afk_element_usage_add(&values->elusage, element);
 362 }
 363 EXPORT_SYMBOL(mlxsw_afk_values_add_u32);
 364
 365 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values,
 366                               enum mlxsw_afk_element element,
 367                               const char *key_value, const char *mask_value,
 368                               unsigned int len)
 369 {
 370         const struct mlxsw_afk_element_info *elinfo =
 371                                 &mlxsw_afk_element_infos[element];
 372         const struct mlxsw_item *storage_item = &elinfo->item;
 373
 374         if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */
 375                 return;
 376         if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) ||
 377             WARN_ON(elinfo->item.size.bytes != len))
 378                 return;
 379         __mlxsw_item_memcpy_to(values->storage.key, key_value,
 380                                storage_item, 0);
 381         __mlxsw_item_memcpy_to(values->storage.mask, mask_value,
 382                                storage_item, 0);
 383         mlxsw_afk_element_usage_add(&values->elusage, element);
 384 }
 385 EXPORT_SYMBOL(mlxsw_afk_values_add_buf);
 386
 387 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item,
 388                                     const struct mlxsw_item *output_item,
 389                                     char *storage, char *output, int diff)
 390 {
 391         u32 value;
 392
 393         value = __mlxsw_item_get32(storage, storage_item, 0);
 394         __mlxsw_item_set32(output, output_item, 0, value + diff);
 395 }
 396
 397 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item,
 398                                     const struct mlxsw_item *output_item,
 399                                     char *storage, char *output)
 400 {
 401         char *storage_data = __mlxsw_item_data(storage, storage_item, 0);
 402         char *output_data = __mlxsw_item_data(output, output_item, 0);
 403         size_t len = output_item->size.bytes;
 404
 405         memcpy(output_data, storage_data, len);
 406 }
 407
 408 static void
 409 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst,
 410                         char *output, char *storage, int u32_diff)
 411 {
 412         const struct mlxsw_item *storage_item = &elinst->info->item;
 413         const struct mlxsw_item *output_item = &elinst->item;
 414
 415         if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32)
 416                 mlxsw_sp_afk_encode_u32(storage_item, output_item,
 417                                         storage, output, u32_diff);
 418         else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF)
 419                 mlxsw_sp_afk_encode_buf(storage_item, output_item,
 420                                         storage, output);
 421 }
 422
 423 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16
 424
 425 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk,
 426                       struct mlxsw_afk_key_info *key_info,
 427                       struct mlxsw_afk_element_values *values,
 428                       char *key, char *mask)
 429 {
 430         unsigned int blocks_count =
 431                         mlxsw_afk_key_info_blocks_count_get(key_info);
 432         char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
 433         char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
 434         const struct mlxsw_afk_element_inst *elinst;
 435         enum mlxsw_afk_element element;
 436         int block_index, i;
 437
 438         for (i = 0; i < blocks_count; i++) {
 439                 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
 440                 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
 441
 442                 mlxsw_afk_element_usage_for_each(element, &values->elusage) {
 443                         elinst = mlxsw_afk_key_info_elinst_get(key_info,
 444                                                                element,
 445                                                                &block_index);
 446                         if (!elinst || block_index != i)
 447                                 continue;
 448
 449                         mlxsw_sp_afk_encode_one(elinst, block_key,
 450                                                 values->storage.key,
 451                                                 elinst->u32_key_diff);
 452                         mlxsw_sp_afk_encode_one(elinst, block_mask,
 453                                                 values->storage.mask, 0);
 454                 }
 455
 456                 mlxsw_afk->ops->encode_block(key, i, block_key);
 457                 mlxsw_afk->ops->encode_block(mask, i, block_mask);
 458         }
 459 }
 460 EXPORT_SYMBOL(mlxsw_afk_encode);
 461
 462 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key,
 463                      int block_start, int block_end)
 464 {
 465         int i;
 466
 467         for (i = block_start; i <= block_end; i++)
 468                 mlxsw_afk->ops->clear_block(key, i);
 469 }
 470 EXPORT_SYMBOL(mlxsw_afk_clear);