drivers/net/ethernet/netronome/nfp/bpf/cmsg.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
   3
   4 #include <linux/bpf.h>
   5 #include <linux/bitops.h>
   6 #include <linux/bug.h>
   7 #include <linux/jiffies.h>
   8 #include <linux/skbuff.h>
   9 #include <linux/timekeeping.h>
  10
  11 #include "../ccm.h"
  12 #include "../nfp_app.h"
  13 #include "../nfp_net.h"
  14 #include "fw.h"
  15 #include "main.h"
  16
  17 static struct sk_buff *
  18 nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size)
  19 {
  20         struct sk_buff *skb;
  21
  22         skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL);
  23         skb_put(skb, size);
  24
  25         return skb;
  26 }
  27
  28 static unsigned int
  29 nfp_bpf_cmsg_map_req_size(struct nfp_app_bpf *bpf, unsigned int n)
  30 {
  31         unsigned int size;
  32
  33         size = sizeof(struct cmsg_req_map_op);
  34         size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
  35
  36         return size;
  37 }
  38
  39 static struct sk_buff *
  40 nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
  41 {
  42         return nfp_bpf_cmsg_alloc(bpf, nfp_bpf_cmsg_map_req_size(bpf, n));
  43 }
  44
  45 static unsigned int
  46 nfp_bpf_cmsg_map_reply_size(struct nfp_app_bpf *bpf, unsigned int n)
  47 {
  48         unsigned int size;
  49
  50         size = sizeof(struct cmsg_reply_map_op);
  51         size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
  52
  53         return size;
  54 }
  55
  56 static int
  57 nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf,
  58                          struct cmsg_reply_map_simple *reply)
  59 {
  60         static const int res_table[] = {
  61                 [CMSG_RC_SUCCESS]       = 0,
  62                 [CMSG_RC_ERR_MAP_FD]    = -EBADFD,
  63                 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT,
  64                 [CMSG_RC_ERR_MAP_ERR]   = -EINVAL,
  65                 [CMSG_RC_ERR_MAP_PARSE] = -EIO,
  66                 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST,
  67                 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM,
  68                 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG,
  69         };
  70         u32 rc;
  71
  72         rc = be32_to_cpu(reply->rc);
  73         if (rc >= ARRAY_SIZE(res_table)) {
  74                 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc);
  75                 return -EIO;
  76         }
  77
  78         return res_table[rc];
  79 }
  80
  81 long long int
  82 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map)
  83 {
  84         struct cmsg_reply_map_alloc_tbl *reply;
  85         struct cmsg_req_map_alloc_tbl *req;
  86         struct sk_buff *skb;
  87         u32 tid;
  88         int err;
  89
  90         skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
  91         if (!skb)
  92                 return -ENOMEM;
  93
  94         req = (void *)skb->data;
  95         req->key_size = cpu_to_be32(map->key_size);
  96         req->value_size = cpu_to_be32(map->value_size);
  97         req->max_entries = cpu_to_be32(map->max_entries);
  98         req->map_type = cpu_to_be32(map->map_type);
  99         req->map_flags = 0;
 100
 101         skb = nfp_ccm_communicate(&bpf->ccm, skb, NFP_CCM_TYPE_BPF_MAP_ALLOC,
 102                                   sizeof(*reply));
 103         if (IS_ERR(skb))
 104                 return PTR_ERR(skb);
 105
 106         reply = (void *)skb->data;
 107         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 108         if (err)
 109                 goto err_free;
 110
 111         tid = be32_to_cpu(reply->tid);
 112         dev_consume_skb_any(skb);
 113
 114         return tid;
 115 err_free:
 116         dev_kfree_skb_any(skb);
 117         return err;
 118 }
 119
 120 void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map)
 121 {
 122         struct cmsg_reply_map_free_tbl *reply;
 123         struct cmsg_req_map_free_tbl *req;
 124         struct sk_buff *skb;
 125         int err;
 126
 127         skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
 128         if (!skb) {
 129                 cmsg_warn(bpf, "leaking map - failed to allocate msg\n");
 130                 return;
 131         }
 132
 133         req = (void *)skb->data;
 134         req->tid = cpu_to_be32(nfp_map->tid);
 135
 136         skb = nfp_ccm_communicate(&bpf->ccm, skb, NFP_CCM_TYPE_BPF_MAP_FREE,
 137                                   sizeof(*reply));
 138         if (IS_ERR(skb)) {
 139                 cmsg_warn(bpf, "leaking map - I/O error\n");
 140                 return;
 141         }
 142
 143         reply = (void *)skb->data;
 144         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 145         if (err)
 146                 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err);
 147
 148         dev_consume_skb_any(skb);
 149 }
 150
 151 static void *
 152 nfp_bpf_ctrl_req_key(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
 153                      unsigned int n)
 154 {
 155         return &req->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
 156 }
 157
 158 static void *
 159 nfp_bpf_ctrl_req_val(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
 160                      unsigned int n)
 161 {
 162         return &req->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
 163 }
 164
 165 static void *
 166 nfp_bpf_ctrl_reply_key(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
 167                        unsigned int n)
 168 {
 169         return &reply->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
 170 }
 171
 172 static void *
 173 nfp_bpf_ctrl_reply_val(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
 174                        unsigned int n)
 175 {
 176         return &reply->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
 177 }
 178
 179 static bool nfp_bpf_ctrl_op_cache_invalidate(enum nfp_ccm_type op)
 180 {
 181         return op == NFP_CCM_TYPE_BPF_MAP_UPDATE ||
 182                op == NFP_CCM_TYPE_BPF_MAP_DELETE;
 183 }
 184
 185 static bool nfp_bpf_ctrl_op_cache_capable(enum nfp_ccm_type op)
 186 {
 187         return op == NFP_CCM_TYPE_BPF_MAP_LOOKUP ||
 188                op == NFP_CCM_TYPE_BPF_MAP_GETNEXT;
 189 }
 190
 191 static bool nfp_bpf_ctrl_op_cache_fill(enum nfp_ccm_type op)
 192 {
 193         return op == NFP_CCM_TYPE_BPF_MAP_GETFIRST ||
 194                op == NFP_CCM_TYPE_BPF_MAP_GETNEXT;
 195 }
 196
 197 static unsigned int
 198 nfp_bpf_ctrl_op_cache_get(struct nfp_bpf_map *nfp_map, enum nfp_ccm_type op,
 199                           const u8 *key, u8 *out_key, u8 *out_value,
 200                           u32 *cache_gen)
 201 {
 202         struct bpf_map *map = &nfp_map->offmap->map;
 203         struct nfp_app_bpf *bpf = nfp_map->bpf;
 204         unsigned int i, count, n_entries;
 205         struct cmsg_reply_map_op *reply;
 206
 207         n_entries = nfp_bpf_ctrl_op_cache_fill(op) ? bpf->cmsg_cache_cnt : 1;
 208
 209         spin_lock(&nfp_map->cache_lock);
 210         *cache_gen = nfp_map->cache_gen;
 211         if (nfp_map->cache_blockers)
 212                 n_entries = 1;
 213
 214         if (nfp_bpf_ctrl_op_cache_invalidate(op))
 215                 goto exit_block;
 216         if (!nfp_bpf_ctrl_op_cache_capable(op))
 217                 goto exit_unlock;
 218
 219         if (!nfp_map->cache)
 220                 goto exit_unlock;
 221         if (nfp_map->cache_to < ktime_get_ns())
 222                 goto exit_invalidate;
 223
 224         reply = (void *)nfp_map->cache->data;
 225         count = be32_to_cpu(reply->count);
 226
 227         for (i = 0; i < count; i++) {
 228                 void *cached_key;
 229
 230                 cached_key = nfp_bpf_ctrl_reply_key(bpf, reply, i);
 231                 if (memcmp(cached_key, key, map->key_size))
 232                         continue;
 233
 234                 if (op == NFP_CCM_TYPE_BPF_MAP_LOOKUP)
 235                         memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, i),
 236                                map->value_size);
 237                 if (op == NFP_CCM_TYPE_BPF_MAP_GETNEXT) {
 238                         if (i + 1 == count)
 239                                 break;
 240
 241                         memcpy(out_key,
 242                                nfp_bpf_ctrl_reply_key(bpf, reply, i + 1),
 243                                map->key_size);
 244                 }
 245
 246                 n_entries = 0;
 247                 goto exit_unlock;
 248         }
 249         goto exit_unlock;
 250
 251 exit_block:
 252         nfp_map->cache_blockers++;
 253 exit_invalidate:
 254         dev_consume_skb_any(nfp_map->cache);
 255         nfp_map->cache = NULL;
 256 exit_unlock:
 257         spin_unlock(&nfp_map->cache_lock);
 258         return n_entries;
 259 }
 260
 261 static void
 262 nfp_bpf_ctrl_op_cache_put(struct nfp_bpf_map *nfp_map, enum nfp_ccm_type op,
 263                           struct sk_buff *skb, u32 cache_gen)
 264 {
 265         bool blocker, filler;
 266
 267         blocker = nfp_bpf_ctrl_op_cache_invalidate(op);
 268         filler = nfp_bpf_ctrl_op_cache_fill(op);
 269         if (blocker || filler) {
 270                 u64 to = 0;
 271
 272                 if (filler)
 273                         to = ktime_get_ns() + NFP_BPF_MAP_CACHE_TIME_NS;
 274
 275                 spin_lock(&nfp_map->cache_lock);
 276                 if (blocker) {
 277                         nfp_map->cache_blockers--;
 278                         nfp_map->cache_gen++;
 279                 }
 280                 if (filler && !nfp_map->cache_blockers &&
 281                     nfp_map->cache_gen == cache_gen) {
 282                         nfp_map->cache_to = to;
 283                         swap(nfp_map->cache, skb);
 284                 }
 285                 spin_unlock(&nfp_map->cache_lock);
 286         }
 287
 288         dev_consume_skb_any(skb);
 289 }
 290
 291 static int
 292 nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap, enum nfp_ccm_type op,
 293                       u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value)
 294 {
 295         struct nfp_bpf_map *nfp_map = offmap->dev_priv;
 296         unsigned int n_entries, reply_entries, count;
 297         struct nfp_app_bpf *bpf = nfp_map->bpf;
 298         struct bpf_map *map = &offmap->map;
 299         struct cmsg_reply_map_op *reply;
 300         struct cmsg_req_map_op *req;
 301         struct sk_buff *skb;
 302         u32 cache_gen;
 303         int err;
 304
 305         /* FW messages have no space for more than 32 bits of flags */
 306         if (flags >> 32)
 307                 return -EOPNOTSUPP;
 308
 309         /* Handle op cache */
 310         n_entries = nfp_bpf_ctrl_op_cache_get(nfp_map, op, key, out_key,
 311                                               out_value, &cache_gen);
 312         if (!n_entries)
 313                 return 0;
 314
 315         skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1);
 316         if (!skb) {
 317                 err = -ENOMEM;
 318                 goto err_cache_put;
 319         }
 320
 321         req = (void *)skb->data;
 322         req->tid = cpu_to_be32(nfp_map->tid);
 323         req->count = cpu_to_be32(n_entries);
 324         req->flags = cpu_to_be32(flags);
 325
 326         /* Copy inputs */
 327         if (key)
 328                 memcpy(nfp_bpf_ctrl_req_key(bpf, req, 0), key, map->key_size);
 329         if (value)
 330                 memcpy(nfp_bpf_ctrl_req_val(bpf, req, 0), value,
 331                        map->value_size);
 332
 333         skb = nfp_ccm_communicate(&bpf->ccm, skb, op, 0);
 334         if (IS_ERR(skb)) {
 335                 err = PTR_ERR(skb);
 336                 goto err_cache_put;
 337         }
 338
 339         if (skb->len < sizeof(*reply)) {
 340                 cmsg_warn(bpf, "cmsg drop - type 0x%02x too short %d!\n",
 341                           op, skb->len);
 342                 err = -EIO;
 343                 goto err_free;
 344         }
 345
 346         reply = (void *)skb->data;
 347         count = be32_to_cpu(reply->count);
 348         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 349         /* FW responds with message sized to hold the good entries,
 350          * plus one extra entry if there was an error.
 351          */
 352         reply_entries = count + !!err;
 353         if (n_entries > 1 && count)
 354                 err = 0;
 355         if (err)
 356                 goto err_free;
 357
 358         if (skb->len != nfp_bpf_cmsg_map_reply_size(bpf, reply_entries)) {
 359                 cmsg_warn(bpf, "cmsg drop - type 0x%02x too short %d for %d entries!\n",
 360                           op, skb->len, reply_entries);
 361                 err = -EIO;
 362                 goto err_free;
 363         }
 364
 365         /* Copy outputs */
 366         if (out_key)
 367                 memcpy(out_key, nfp_bpf_ctrl_reply_key(bpf, reply, 0),
 368                        map->key_size);
 369         if (out_value)
 370                 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, 0),
 371                        map->value_size);
 372
 373         nfp_bpf_ctrl_op_cache_put(nfp_map, op, skb, cache_gen);
 374
 375         return 0;
 376 err_free:
 377         dev_kfree_skb_any(skb);
 378 err_cache_put:
 379         nfp_bpf_ctrl_op_cache_put(nfp_map, op, NULL, cache_gen);
 380         return err;
 381 }
 382
 383 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
 384                               void *key, void *value, u64 flags)
 385 {
 386         return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_UPDATE,
 387                                      key, value, flags, NULL, NULL);
 388 }
 389
 390 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key)
 391 {
 392         return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_DELETE,
 393                                      key, NULL, 0, NULL, NULL);
 394 }
 395
 396 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
 397                               void *key, void *value)
 398 {
 399         return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_LOOKUP,
 400                                      key, NULL, 0, NULL, value);
 401 }
 402
 403 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
 404                                 void *next_key)
 405 {
 406         return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_GETFIRST,
 407                                      NULL, NULL, 0, next_key, NULL);
 408 }
 409
 410 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
 411                                void *key, void *next_key)
 412 {
 413         return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_GETNEXT,
 414                                      key, NULL, 0, next_key, NULL);
 415 }
 416
 417 unsigned int nfp_bpf_ctrl_cmsg_min_mtu(struct nfp_app_bpf *bpf)
 418 {
 419         return max(nfp_bpf_cmsg_map_req_size(bpf, 1),
 420                    nfp_bpf_cmsg_map_reply_size(bpf, 1));
 421 }
 422
 423 unsigned int nfp_bpf_ctrl_cmsg_mtu(struct nfp_app_bpf *bpf)
 424 {
 425         return max3(NFP_NET_DEFAULT_MTU,
 426                     nfp_bpf_cmsg_map_req_size(bpf, NFP_BPF_MAP_CACHE_CNT),
 427                     nfp_bpf_cmsg_map_reply_size(bpf, NFP_BPF_MAP_CACHE_CNT));
 428 }
 429
 430 unsigned int nfp_bpf_ctrl_cmsg_cache_cnt(struct nfp_app_bpf *bpf)
 431 {
 432         unsigned int mtu, req_max, reply_max, entry_sz;
 433
 434         mtu = bpf->app->ctrl->dp.mtu;
 435         entry_sz = bpf->cmsg_key_sz + bpf->cmsg_val_sz;
 436         req_max = (mtu - sizeof(struct cmsg_req_map_op)) / entry_sz;
 437         reply_max = (mtu - sizeof(struct cmsg_reply_map_op)) / entry_sz;
 438
 439         return min3(req_max, reply_max, NFP_BPF_MAP_CACHE_CNT);
 440 }
 441
 442 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
 443 {
 444         struct nfp_app_bpf *bpf = app->priv;
 445
 446         if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) {
 447                 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len);
 448                 dev_kfree_skb_any(skb);
 449                 return;
 450         }
 451
 452         if (nfp_ccm_get_type(skb) == NFP_CCM_TYPE_BPF_BPF_EVENT) {
 453                 if (!nfp_bpf_event_output(bpf, skb->data, skb->len))
 454                         dev_consume_skb_any(skb);
 455                 else
 456                         dev_kfree_skb_any(skb);
 457         }
 458
 459         nfp_ccm_rx(&bpf->ccm, skb);
 460 }
 461
 462 void
 463 nfp_bpf_ctrl_msg_rx_raw(struct nfp_app *app, const void *data, unsigned int len)
 464 {
 465         const struct nfp_ccm_hdr *hdr = data;
 466         struct nfp_app_bpf *bpf = app->priv;
 467
 468         if (unlikely(len < sizeof(struct cmsg_reply_map_simple))) {
 469                 cmsg_warn(bpf, "cmsg drop - too short %d!\n", len);
 470                 return;
 471         }
 472
 473         if (hdr->type == NFP_CCM_TYPE_BPF_BPF_EVENT)
 474                 nfp_bpf_event_output(bpf, data, len);
 475         else
 476                 cmsg_warn(bpf, "cmsg drop - msg type %d with raw buffer!\n",
 477                           hdr->type);
 478 }