tools/perf/util/bpf-loader.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * bpf-loader.c
   4  *
   5  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   6  * Copyright (C) 2015 Huawei Inc.
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <bpf/libbpf.h>
  11 #include <bpf/bpf.h>
  12 #include <linux/err.h>
  13 #include <linux/kernel.h>
  14 #include <linux/string.h>
  15 #include <linux/zalloc.h>
  16 #include <errno.h>
  17 #include "perf.h"
  18 #include "debug.h"
  19 #include "evlist.h"
  20 #include "bpf-loader.h"
  21 #include "bpf-prologue.h"
  22 #include "probe-event.h"
  23 #include "probe-finder.h" // for MAX_PROBES
  24 #include "parse-events.h"
  25 #include "strfilter.h"
  26 #include "llvm-utils.h"
  27 #include "c++/clang-c.h"
  28
  29 static int libbpf_perf_print(enum libbpf_print_level level __attribute__((unused)),
  30                               const char *fmt, va_list args)
  31 {
  32         return veprintf(1, verbose, pr_fmt(fmt), args);
  33 }
  34
  35 struct bpf_prog_priv {
  36         bool is_tp;
  37         char *sys_name;
  38         char *evt_name;
  39         struct perf_probe_event pev;
  40         bool need_prologue;
  41         struct bpf_insn *insns_buf;
  42         int nr_types;
  43         int *type_mapping;
  44 };
  45
  46 static bool libbpf_initialized;
  47
  48 struct bpf_object *
  49 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  50 {
  51         struct bpf_object *obj;
  52
  53         if (!libbpf_initialized) {
  54                 libbpf_set_print(libbpf_perf_print);
  55                 libbpf_initialized = true;
  56         }
  57
  58         obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
  59         if (IS_ERR_OR_NULL(obj)) {
  60                 pr_debug("bpf: failed to load buffer\n");
  61                 return ERR_PTR(-EINVAL);
  62         }
  63
  64         return obj;
  65 }
  66
  67 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  68 {
  69         struct bpf_object *obj;
  70
  71         if (!libbpf_initialized) {
  72                 libbpf_set_print(libbpf_perf_print);
  73                 libbpf_initialized = true;
  74         }
  75
  76         if (source) {
  77                 int err;
  78                 void *obj_buf;
  79                 size_t obj_buf_sz;
  80
  81                 perf_clang__init();
  82                 err = perf_clang__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  83                 perf_clang__cleanup();
  84                 if (err) {
  85                         pr_debug("bpf: builtin compilation failed: %d, try external compiler\n", err);
  86                         err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  87                         if (err)
  88                                 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  89                 } else
  90                         pr_debug("bpf: successful builtin compilation\n");
  91                 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
  92
  93                 if (!IS_ERR_OR_NULL(obj) && llvm_param.dump_obj)
  94                         llvm__dump_obj(filename, obj_buf, obj_buf_sz);
  95
  96                 free(obj_buf);
  97         } else
  98                 obj = bpf_object__open(filename);
  99
 100         if (IS_ERR_OR_NULL(obj)) {
 101                 pr_debug("bpf: failed to load %s\n", filename);
 102                 return obj;
 103         }
 104
 105         return obj;
 106 }
 107
 108 void bpf__clear(void)
 109 {
 110         struct bpf_object *obj, *tmp;
 111
 112         bpf_object__for_each_safe(obj, tmp) {
 113                 bpf__unprobe(obj);
 114                 bpf_object__close(obj);
 115         }
 116 }
 117
 118 static void
 119 clear_prog_priv(struct bpf_program *prog __maybe_unused,
 120                 void *_priv)
 121 {
 122         struct bpf_prog_priv *priv = _priv;
 123
 124         cleanup_perf_probe_events(&priv->pev, 1);
 125         zfree(&priv->insns_buf);
 126         zfree(&priv->type_mapping);
 127         zfree(&priv->sys_name);
 128         zfree(&priv->evt_name);
 129         free(priv);
 130 }
 131
 132 static int
 133 prog_config__exec(const char *value, struct perf_probe_event *pev)
 134 {
 135         pev->uprobes = true;
 136         pev->target = strdup(value);
 137         if (!pev->target)
 138                 return -ENOMEM;
 139         return 0;
 140 }
 141
 142 static int
 143 prog_config__module(const char *value, struct perf_probe_event *pev)
 144 {
 145         pev->uprobes = false;
 146         pev->target = strdup(value);
 147         if (!pev->target)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static int
 153 prog_config__bool(const char *value, bool *pbool, bool invert)
 154 {
 155         int err;
 156         bool bool_value;
 157
 158         if (!pbool)
 159                 return -EINVAL;
 160
 161         err = strtobool(value, &bool_value);
 162         if (err)
 163                 return err;
 164
 165         *pbool = invert ? !bool_value : bool_value;
 166         return 0;
 167 }
 168
 169 static int
 170 prog_config__inlines(const char *value,
 171                      struct perf_probe_event *pev __maybe_unused)
 172 {
 173         return prog_config__bool(value, &probe_conf.no_inlines, true);
 174 }
 175
 176 static int
 177 prog_config__force(const char *value,
 178                    struct perf_probe_event *pev __maybe_unused)
 179 {
 180         return prog_config__bool(value, &probe_conf.force_add, false);
 181 }
 182
 183 static struct {
 184         const char *key;
 185         const char *usage;
 186         const char *desc;
 187         int (*func)(const char *, struct perf_probe_event *);
 188 } bpf_prog_config_terms[] = {
 189         {
 190                 .key    = "exec",
 191                 .usage  = "exec=<full path of file>",
 192                 .desc   = "Set uprobe target",
 193                 .func   = prog_config__exec,
 194         },
 195         {
 196                 .key    = "module",
 197                 .usage  = "module=<module name>    ",
 198                 .desc   = "Set kprobe module",
 199                 .func   = prog_config__module,
 200         },
 201         {
 202                 .key    = "inlines",
 203                 .usage  = "inlines=[yes|no]        ",
 204                 .desc   = "Probe at inline symbol",
 205                 .func   = prog_config__inlines,
 206         },
 207         {
 208                 .key    = "force",
 209                 .usage  = "force=[yes|no]          ",
 210                 .desc   = "Forcibly add events with existing name",
 211                 .func   = prog_config__force,
 212         },
 213 };
 214
 215 static int
 216 do_prog_config(const char *key, const char *value,
 217                struct perf_probe_event *pev)
 218 {
 219         unsigned int i;
 220
 221         pr_debug("config bpf program: %s=%s\n", key, value);
 222         for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 223                 if (strcmp(key, bpf_prog_config_terms[i].key) == 0)
 224                         return bpf_prog_config_terms[i].func(value, pev);
 225
 226         pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
 227                  key, value);
 228
 229         pr_debug("\nHint: Valid options are:\n");
 230         for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 231                 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage,
 232                          bpf_prog_config_terms[i].desc);
 233         pr_debug("\n");
 234
 235         return -BPF_LOADER_ERRNO__PROGCONF_TERM;
 236 }
 237
 238 static const char *
 239 parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev)
 240 {
 241         char *text = strdup(config_str);
 242         char *sep, *line;
 243         const char *main_str = NULL;
 244         int err = 0;
 245
 246         if (!text) {
 247                 pr_debug("Not enough memory: dup config_str failed\n");
 248                 return ERR_PTR(-ENOMEM);
 249         }
 250
 251         line = text;
 252         while ((sep = strchr(line, ';'))) {
 253                 char *equ;
 254
 255                 *sep = '\0';
 256                 equ = strchr(line, '=');
 257                 if (!equ) {
 258                         pr_warning("WARNING: invalid config in BPF object: %s\n",
 259                                    line);
 260                         pr_warning("\tShould be 'key=value'.\n");
 261                         goto nextline;
 262                 }
 263                 *equ = '\0';
 264
 265                 err = do_prog_config(line, equ + 1, pev);
 266                 if (err)
 267                         break;
 268 nextline:
 269                 line = sep + 1;
 270         }
 271
 272         if (!err)
 273                 main_str = config_str + (line - text);
 274         free(text);
 275
 276         return err ? ERR_PTR(err) : main_str;
 277 }
 278
 279 static int
 280 parse_prog_config(const char *config_str, const char **p_main_str,
 281                   bool *is_tp, struct perf_probe_event *pev)
 282 {
 283         int err;
 284         const char *main_str = parse_prog_config_kvpair(config_str, pev);
 285
 286         if (IS_ERR(main_str))
 287                 return PTR_ERR(main_str);
 288
 289         *p_main_str = main_str;
 290         if (!strchr(main_str, '=')) {
 291                 /* Is a tracepoint event? */
 292                 const char *s = strchr(main_str, ':');
 293
 294                 if (!s) {
 295                         pr_debug("bpf: '%s' is not a valid tracepoint\n",
 296                                  config_str);
 297                         return -BPF_LOADER_ERRNO__CONFIG;
 298                 }
 299
 300                 *is_tp = true;
 301                 return 0;
 302         }
 303
 304         *is_tp = false;
 305         err = parse_perf_probe_command(main_str, pev);
 306         if (err < 0) {
 307                 pr_debug("bpf: '%s' is not a valid config string\n",
 308                          config_str);
 309                 /* parse failed, don't need clear pev. */
 310                 return -BPF_LOADER_ERRNO__CONFIG;
 311         }
 312         return 0;
 313 }
 314
 315 static int
 316 config_bpf_program(struct bpf_program *prog)
 317 {
 318         struct perf_probe_event *pev = NULL;
 319         struct bpf_prog_priv *priv = NULL;
 320         const char *config_str, *main_str;
 321         bool is_tp = false;
 322         int err;
 323
 324         /* Initialize per-program probing setting */
 325         probe_conf.no_inlines = false;
 326         probe_conf.force_add = false;
 327
 328         config_str = bpf_program__title(prog, false);
 329         if (IS_ERR(config_str)) {
 330                 pr_debug("bpf: unable to get title for program\n");
 331                 return PTR_ERR(config_str);
 332         }
 333
 334         priv = calloc(sizeof(*priv), 1);
 335         if (!priv) {
 336                 pr_debug("bpf: failed to alloc priv\n");
 337                 return -ENOMEM;
 338         }
 339         pev = &priv->pev;
 340
 341         pr_debug("bpf: config program '%s'\n", config_str);
 342         err = parse_prog_config(config_str, &main_str, &is_tp, pev);
 343         if (err)
 344                 goto errout;
 345
 346         if (is_tp) {
 347                 char *s = strchr(main_str, ':');
 348
 349                 priv->is_tp = true;
 350                 priv->sys_name = strndup(main_str, s - main_str);
 351                 priv->evt_name = strdup(s + 1);
 352                 goto set_priv;
 353         }
 354
 355         if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 356                 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 357                          config_str, PERF_BPF_PROBE_GROUP);
 358                 err = -BPF_LOADER_ERRNO__GROUP;
 359                 goto errout;
 360         } else if (!pev->group)
 361                 pev->group = strdup(PERF_BPF_PROBE_GROUP);
 362
 363         if (!pev->group) {
 364                 pr_debug("bpf: strdup failed\n");
 365                 err = -ENOMEM;
 366                 goto errout;
 367         }
 368
 369         if (!pev->event) {
 370                 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 371                          config_str);
 372                 err = -BPF_LOADER_ERRNO__EVENTNAME;
 373                 goto errout;
 374         }
 375         pr_debug("bpf: config '%s' is ok\n", config_str);
 376
 377 set_priv:
 378         err = bpf_program__set_priv(prog, priv, clear_prog_priv);
 379         if (err) {
 380                 pr_debug("Failed to set priv for program '%s'\n", config_str);
 381                 goto errout;
 382         }
 383
 384         return 0;
 385
 386 errout:
 387         if (pev)
 388                 clear_perf_probe_event(pev);
 389         free(priv);
 390         return err;
 391 }
 392
 393 static int bpf__prepare_probe(void)
 394 {
 395         static int err = 0;
 396         static bool initialized = false;
 397
 398         /*
 399          * Make err static, so if init failed the first, bpf__prepare_probe()
 400          * fails each time without calling init_probe_symbol_maps multiple
 401          * times.
 402          */
 403         if (initialized)
 404                 return err;
 405
 406         initialized = true;
 407         err = init_probe_symbol_maps(false);
 408         if (err < 0)
 409                 pr_debug("Failed to init_probe_symbol_maps\n");
 410         probe_conf.max_probes = MAX_PROBES;
 411         return err;
 412 }
 413
 414 static int
 415 preproc_gen_prologue(struct bpf_program *prog, int n,
 416                      struct bpf_insn *orig_insns, int orig_insns_cnt,
 417                      struct bpf_prog_prep_result *res)
 418 {
 419         struct bpf_prog_priv *priv = bpf_program__priv(prog);
 420         struct probe_trace_event *tev;
 421         struct perf_probe_event *pev;
 422         struct bpf_insn *buf;
 423         size_t prologue_cnt = 0;
 424         int i, err;
 425
 426         if (IS_ERR(priv) || !priv || priv->is_tp)
 427                 goto errout;
 428
 429         pev = &priv->pev;
 430
 431         if (n < 0 || n >= priv->nr_types)
 432                 goto errout;
 433
 434         /* Find a tev belongs to that type */
 435         for (i = 0; i < pev->ntevs; i++) {
 436                 if (priv->type_mapping[i] == n)
 437                         break;
 438         }
 439
 440         if (i >= pev->ntevs) {
 441                 pr_debug("Internal error: prologue type %d not found\n", n);
 442                 return -BPF_LOADER_ERRNO__PROLOGUE;
 443         }
 444
 445         tev = &pev->tevs[i];
 446
 447         buf = priv->insns_buf;
 448         err = bpf__gen_prologue(tev->args, tev->nargs,
 449                                 buf, &prologue_cnt,
 450                                 BPF_MAXINSNS - orig_insns_cnt);
 451         if (err) {
 452                 const char *title;
 453
 454                 title = bpf_program__title(prog, false);
 455                 if (!title)
 456                         title = "[unknown]";
 457
 458                 pr_debug("Failed to generate prologue for program %s\n",
 459                          title);
 460                 return err;
 461         }
 462
 463         memcpy(&buf[prologue_cnt], orig_insns,
 464                sizeof(struct bpf_insn) * orig_insns_cnt);
 465
 466         res->new_insn_ptr = buf;
 467         res->new_insn_cnt = prologue_cnt + orig_insns_cnt;
 468         res->pfd = NULL;
 469         return 0;
 470
 471 errout:
 472         pr_debug("Internal error in preproc_gen_prologue\n");
 473         return -BPF_LOADER_ERRNO__PROLOGUE;
 474 }
 475
 476 /*
 477  * compare_tev_args is reflexive, transitive and antisymmetric.
 478  * I can proof it but this margin is too narrow to contain.
 479  */
 480 static int compare_tev_args(const void *ptev1, const void *ptev2)
 481 {
 482         int i, ret;
 483         const struct probe_trace_event *tev1 =
 484                 *(const struct probe_trace_event **)ptev1;
 485         const struct probe_trace_event *tev2 =
 486                 *(const struct probe_trace_event **)ptev2;
 487
 488         ret = tev2->nargs - tev1->nargs;
 489         if (ret)
 490                 return ret;
 491
 492         for (i = 0; i < tev1->nargs; i++) {
 493                 struct probe_trace_arg *arg1, *arg2;
 494                 struct probe_trace_arg_ref *ref1, *ref2;
 495
 496                 arg1 = &tev1->args[i];
 497                 arg2 = &tev2->args[i];
 498
 499                 ret = strcmp(arg1->value, arg2->value);
 500                 if (ret)
 501                         return ret;
 502
 503                 ref1 = arg1->ref;
 504                 ref2 = arg2->ref;
 505
 506                 while (ref1 && ref2) {
 507                         ret = ref2->offset - ref1->offset;
 508                         if (ret)
 509                                 return ret;
 510
 511                         ref1 = ref1->next;
 512                         ref2 = ref2->next;
 513                 }
 514
 515                 if (ref1 || ref2)
 516                         return ref2 ? 1 : -1;
 517         }
 518
 519         return 0;
 520 }
 521
 522 /*
 523  * Assign a type number to each tevs in a pev.
 524  * mapping is an array with same slots as tevs in that pev.
 525  * nr_types will be set to number of types.
 526  */
 527 static int map_prologue(struct perf_probe_event *pev, int *mapping,
 528                         int *nr_types)
 529 {
 530         int i, type = 0;
 531         struct probe_trace_event **ptevs;
 532
 533         size_t array_sz = sizeof(*ptevs) * pev->ntevs;
 534
 535         ptevs = malloc(array_sz);
 536         if (!ptevs) {
 537                 pr_debug("Not enough memory: alloc ptevs failed\n");
 538                 return -ENOMEM;
 539         }
 540
 541         pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs);
 542         for (i = 0; i < pev->ntevs; i++)
 543                 ptevs[i] = &pev->tevs[i];
 544
 545         qsort(ptevs, pev->ntevs, sizeof(*ptevs),
 546               compare_tev_args);
 547
 548         for (i = 0; i < pev->ntevs; i++) {
 549                 int n;
 550
 551                 n = ptevs[i] - pev->tevs;
 552                 if (i == 0) {
 553                         mapping[n] = type;
 554                         pr_debug("mapping[%d]=%d\n", n, type);
 555                         continue;
 556                 }
 557
 558                 if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0)
 559                         mapping[n] = type;
 560                 else
 561                         mapping[n] = ++type;
 562
 563                 pr_debug("mapping[%d]=%d\n", n, mapping[n]);
 564         }
 565         free(ptevs);
 566         *nr_types = type + 1;
 567
 568         return 0;
 569 }
 570
 571 static int hook_load_preprocessor(struct bpf_program *prog)
 572 {
 573         struct bpf_prog_priv *priv = bpf_program__priv(prog);
 574         struct perf_probe_event *pev;
 575         bool need_prologue = false;
 576         int err, i;
 577
 578         if (IS_ERR(priv) || !priv) {
 579                 pr_debug("Internal error when hook preprocessor\n");
 580                 return -BPF_LOADER_ERRNO__INTERNAL;
 581         }
 582
 583         if (priv->is_tp) {
 584                 priv->need_prologue = false;
 585                 return 0;
 586         }
 587
 588         pev = &priv->pev;
 589         for (i = 0; i < pev->ntevs; i++) {
 590                 struct probe_trace_event *tev = &pev->tevs[i];
 591
 592                 if (tev->nargs > 0) {
 593                         need_prologue = true;
 594                         break;
 595                 }
 596         }
 597
 598         /*
 599          * Since all tevs don't have argument, we don't need generate
 600          * prologue.
 601          */
 602         if (!need_prologue) {
 603                 priv->need_prologue = false;
 604                 return 0;
 605         }
 606
 607         priv->need_prologue = true;
 608         priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS);
 609         if (!priv->insns_buf) {
 610                 pr_debug("Not enough memory: alloc insns_buf failed\n");
 611                 return -ENOMEM;
 612         }
 613
 614         priv->type_mapping = malloc(sizeof(int) * pev->ntevs);
 615         if (!priv->type_mapping) {
 616                 pr_debug("Not enough memory: alloc type_mapping failed\n");
 617                 return -ENOMEM;
 618         }
 619         memset(priv->type_mapping, -1,
 620                sizeof(int) * pev->ntevs);
 621
 622         err = map_prologue(pev, priv->type_mapping, &priv->nr_types);
 623         if (err)
 624                 return err;
 625
 626         err = bpf_program__set_prep(prog, priv->nr_types,
 627                                     preproc_gen_prologue);
 628         return err;
 629 }
 630
 631 int bpf__probe(struct bpf_object *obj)
 632 {
 633         int err = 0;
 634         struct bpf_program *prog;
 635         struct bpf_prog_priv *priv;
 636         struct perf_probe_event *pev;
 637
 638         err = bpf__prepare_probe();
 639         if (err) {
 640                 pr_debug("bpf__prepare_probe failed\n");
 641                 return err;
 642         }
 643
 644         bpf_object__for_each_program(prog, obj) {
 645                 err = config_bpf_program(prog);
 646                 if (err)
 647                         goto out;
 648
 649                 priv = bpf_program__priv(prog);
 650                 if (IS_ERR(priv) || !priv) {
 651                         err = PTR_ERR(priv);
 652                         goto out;
 653                 }
 654
 655                 if (priv->is_tp) {
 656                         bpf_program__set_tracepoint(prog);
 657                         continue;
 658                 }
 659
 660                 bpf_program__set_kprobe(prog);
 661                 pev = &priv->pev;
 662
 663                 err = convert_perf_probe_events(pev, 1);
 664                 if (err < 0) {
 665                         pr_debug("bpf_probe: failed to convert perf probe events\n");
 666                         goto out;
 667                 }
 668
 669                 err = apply_perf_probe_events(pev, 1);
 670                 if (err < 0) {
 671                         pr_debug("bpf_probe: failed to apply perf probe events\n");
 672                         goto out;
 673                 }
 674
 675                 /*
 676                  * After probing, let's consider prologue, which
 677                  * adds program fetcher to BPF programs.
 678                  *
 679                  * hook_load_preprocessorr() hooks pre-processor
 680                  * to bpf_program, let it generate prologue
 681                  * dynamically during loading.
 682                  */
 683                 err = hook_load_preprocessor(prog);
 684                 if (err)
 685                         goto out;
 686         }
 687 out:
 688         return err < 0 ? err : 0;
 689 }
 690
 691 #define EVENTS_WRITE_BUFSIZE  4096
 692 int bpf__unprobe(struct bpf_object *obj)
 693 {
 694         int err, ret = 0;
 695         struct bpf_program *prog;
 696
 697         bpf_object__for_each_program(prog, obj) {
 698                 struct bpf_prog_priv *priv = bpf_program__priv(prog);
 699                 int i;
 700
 701                 if (IS_ERR(priv) || !priv || priv->is_tp)
 702                         continue;
 703
 704                 for (i = 0; i < priv->pev.ntevs; i++) {
 705                         struct probe_trace_event *tev = &priv->pev.tevs[i];
 706                         char name_buf[EVENTS_WRITE_BUFSIZE];
 707                         struct strfilter *delfilter;
 708
 709                         snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 710                                  "%s:%s", tev->group, tev->event);
 711                         name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 712
 713                         delfilter = strfilter__new(name_buf, NULL);
 714                         if (!delfilter) {
 715                                 pr_debug("Failed to create filter for unprobing\n");
 716                                 ret = -ENOMEM;
 717                                 continue;
 718                         }
 719
 720                         err = del_perf_probe_events(delfilter);
 721                         strfilter__delete(delfilter);
 722                         if (err) {
 723                                 pr_debug("Failed to delete %s\n", name_buf);
 724                                 ret = err;
 725                                 continue;
 726                         }
 727                 }
 728         }
 729         return ret;
 730 }
 731
 732 int bpf__load(struct bpf_object *obj)
 733 {
 734         int err;
 735
 736         err = bpf_object__load(obj);
 737         if (err) {
 738                 char bf[128];
 739                 libbpf_strerror(err, bf, sizeof(bf));
 740                 pr_debug("bpf: load objects failed: err=%d: (%s)\n", err, bf);
 741                 return err;
 742         }
 743         return 0;
 744 }
 745
 746 int bpf__foreach_event(struct bpf_object *obj,
 747                        bpf_prog_iter_callback_t func,
 748                        void *arg)
 749 {
 750         struct bpf_program *prog;
 751         int err;
 752
 753         bpf_object__for_each_program(prog, obj) {
 754                 struct bpf_prog_priv *priv = bpf_program__priv(prog);
 755                 struct probe_trace_event *tev;
 756                 struct perf_probe_event *pev;
 757                 int i, fd;
 758
 759                 if (IS_ERR(priv) || !priv) {
 760                         pr_debug("bpf: failed to get private field\n");
 761                         return -BPF_LOADER_ERRNO__INTERNAL;
 762                 }
 763
 764                 if (priv->is_tp) {
 765                         fd = bpf_program__fd(prog);
 766                         err = (*func)(priv->sys_name, priv->evt_name, fd, arg);
 767                         if (err) {
 768                                 pr_debug("bpf: tracepoint call back failed, stop iterate\n");
 769                                 return err;
 770                         }
 771                         continue;
 772                 }
 773
 774                 pev = &priv->pev;
 775                 for (i = 0; i < pev->ntevs; i++) {
 776                         tev = &pev->tevs[i];
 777
 778                         if (priv->need_prologue) {
 779                                 int type = priv->type_mapping[i];
 780
 781                                 fd = bpf_program__nth_fd(prog, type);
 782                         } else {
 783                                 fd = bpf_program__fd(prog);
 784                         }
 785
 786                         if (fd < 0) {
 787                                 pr_debug("bpf: failed to get file descriptor\n");
 788                                 return fd;
 789                         }
 790
 791                         err = (*func)(tev->group, tev->event, fd, arg);
 792                         if (err) {
 793                                 pr_debug("bpf: call back failed, stop iterate\n");
 794                                 return err;
 795                         }
 796                 }
 797         }
 798         return 0;
 799 }
 800
 801 enum bpf_map_op_type {
 802         BPF_MAP_OP_SET_VALUE,
 803         BPF_MAP_OP_SET_EVSEL,
 804 };
 805
 806 enum bpf_map_key_type {
 807         BPF_MAP_KEY_ALL,
 808         BPF_MAP_KEY_RANGES,
 809 };
 810
 811 struct bpf_map_op {
 812         struct list_head list;
 813         enum bpf_map_op_type op_type;
 814         enum bpf_map_key_type key_type;
 815         union {
 816                 struct parse_events_array array;
 817         } k;
 818         union {
 819                 u64 value;
 820                 struct perf_evsel *evsel;
 821         } v;
 822 };
 823
 824 struct bpf_map_priv {
 825         struct list_head ops_list;
 826 };
 827
 828 static void
 829 bpf_map_op__delete(struct bpf_map_op *op)
 830 {
 831         if (!list_empty(&op->list))
 832                 list_del_init(&op->list);
 833         if (op->key_type == BPF_MAP_KEY_RANGES)
 834                 parse_events__clear_array(&op->k.array);
 835         free(op);
 836 }
 837
 838 static void
 839 bpf_map_priv__purge(struct bpf_map_priv *priv)
 840 {
 841         struct bpf_map_op *pos, *n;
 842
 843         list_for_each_entry_safe(pos, n, &priv->ops_list, list) {
 844                 list_del_init(&pos->list);
 845                 bpf_map_op__delete(pos);
 846         }
 847 }
 848
 849 static void
 850 bpf_map_priv__clear(struct bpf_map *map __maybe_unused,
 851                     void *_priv)
 852 {
 853         struct bpf_map_priv *priv = _priv;
 854
 855         bpf_map_priv__purge(priv);
 856         free(priv);
 857 }
 858
 859 static int
 860 bpf_map_op_setkey(struct bpf_map_op *op, struct parse_events_term *term)
 861 {
 862         op->key_type = BPF_MAP_KEY_ALL;
 863         if (!term)
 864                 return 0;
 865
 866         if (term->array.nr_ranges) {
 867                 size_t memsz = term->array.nr_ranges *
 868                                 sizeof(op->k.array.ranges[0]);
 869
 870                 op->k.array.ranges = memdup(term->array.ranges, memsz);
 871                 if (!op->k.array.ranges) {
 872                         pr_debug("Not enough memory to alloc indices for map\n");
 873                         return -ENOMEM;
 874                 }
 875                 op->key_type = BPF_MAP_KEY_RANGES;
 876                 op->k.array.nr_ranges = term->array.nr_ranges;
 877         }
 878         return 0;
 879 }
 880
 881 static struct bpf_map_op *
 882 bpf_map_op__new(struct parse_events_term *term)
 883 {
 884         struct bpf_map_op *op;
 885         int err;
 886
 887         op = zalloc(sizeof(*op));
 888         if (!op) {
 889                 pr_debug("Failed to alloc bpf_map_op\n");
 890                 return ERR_PTR(-ENOMEM);
 891         }
 892         INIT_LIST_HEAD(&op->list);
 893
 894         err = bpf_map_op_setkey(op, term);
 895         if (err) {
 896                 free(op);
 897                 return ERR_PTR(err);
 898         }
 899         return op;
 900 }
 901
 902 static struct bpf_map_op *
 903 bpf_map_op__clone(struct bpf_map_op *op)
 904 {
 905         struct bpf_map_op *newop;
 906
 907         newop = memdup(op, sizeof(*op));
 908         if (!newop) {
 909                 pr_debug("Failed to alloc bpf_map_op\n");
 910                 return NULL;
 911         }
 912
 913         INIT_LIST_HEAD(&newop->list);
 914         if (op->key_type == BPF_MAP_KEY_RANGES) {
 915                 size_t memsz = op->k.array.nr_ranges *
 916                                sizeof(op->k.array.ranges[0]);
 917
 918                 newop->k.array.ranges = memdup(op->k.array.ranges, memsz);
 919                 if (!newop->k.array.ranges) {
 920                         pr_debug("Failed to alloc indices for map\n");
 921                         free(newop);
 922                         return NULL;
 923                 }
 924         }
 925
 926         return newop;
 927 }
 928
 929 static struct bpf_map_priv *
 930 bpf_map_priv__clone(struct bpf_map_priv *priv)
 931 {
 932         struct bpf_map_priv *newpriv;
 933         struct bpf_map_op *pos, *newop;
 934
 935         newpriv = zalloc(sizeof(*newpriv));
 936         if (!newpriv) {
 937                 pr_debug("Not enough memory to alloc map private\n");
 938                 return NULL;
 939         }
 940         INIT_LIST_HEAD(&newpriv->ops_list);
 941
 942         list_for_each_entry(pos, &priv->ops_list, list) {
 943                 newop = bpf_map_op__clone(pos);
 944                 if (!newop) {
 945                         bpf_map_priv__purge(newpriv);
 946                         return NULL;
 947                 }
 948                 list_add_tail(&newop->list, &newpriv->ops_list);
 949         }
 950
 951         return newpriv;
 952 }
 953
 954 static int
 955 bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op)
 956 {
 957         const char *map_name = bpf_map__name(map);
 958         struct bpf_map_priv *priv = bpf_map__priv(map);
 959
 960         if (IS_ERR(priv)) {
 961                 pr_debug("Failed to get private from map %s\n", map_name);
 962                 return PTR_ERR(priv);
 963         }
 964
 965         if (!priv) {
 966                 priv = zalloc(sizeof(*priv));
 967                 if (!priv) {
 968                         pr_debug("Not enough memory to alloc map private\n");
 969                         return -ENOMEM;
 970                 }
 971                 INIT_LIST_HEAD(&priv->ops_list);
 972
 973                 if (bpf_map__set_priv(map, priv, bpf_map_priv__clear)) {
 974                         free(priv);
 975                         return -BPF_LOADER_ERRNO__INTERNAL;
 976                 }
 977         }
 978
 979         list_add_tail(&op->list, &priv->ops_list);
 980         return 0;
 981 }
 982
 983 static struct bpf_map_op *
 984 bpf_map__add_newop(struct bpf_map *map, struct parse_events_term *term)
 985 {
 986         struct bpf_map_op *op;
 987         int err;
 988
 989         op = bpf_map_op__new(term);
 990         if (IS_ERR(op))
 991                 return op;
 992
 993         err = bpf_map__add_op(map, op);
 994         if (err) {
 995                 bpf_map_op__delete(op);
 996                 return ERR_PTR(err);
 997         }
 998         return op;
 999 }
1000
1001 static int
1002 __bpf_map__config_value(struct bpf_map *map,
1003                         struct parse_events_term *term)
1004 {
1005         struct bpf_map_op *op;
1006         const char *map_name = bpf_map__name(map);
1007         const struct bpf_map_def *def = bpf_map__def(map);
1008
1009         if (IS_ERR(def)) {
1010                 pr_debug("Unable to get map definition from '%s'\n",
1011                          map_name);
1012                 return -BPF_LOADER_ERRNO__INTERNAL;
1013         }
1014
1015         if (def->type != BPF_MAP_TYPE_ARRAY) {
1016                 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
1017                          map_name);
1018                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1019         }
1020         if (def->key_size < sizeof(unsigned int)) {
1021                 pr_debug("Map %s has incorrect key size\n", map_name);
1022                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE;
1023         }
1024         switch (def->value_size) {
1025         case 1:
1026         case 2:
1027         case 4:
1028         case 8:
1029                 break;
1030         default:
1031                 pr_debug("Map %s has incorrect value size\n", map_name);
1032                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1033         }
1034
1035         op = bpf_map__add_newop(map, term);
1036         if (IS_ERR(op))
1037                 return PTR_ERR(op);
1038         op->op_type = BPF_MAP_OP_SET_VALUE;
1039         op->v.value = term->val.num;
1040         return 0;
1041 }
1042
1043 static int
1044 bpf_map__config_value(struct bpf_map *map,
1045                       struct parse_events_term *term,
1046                       struct perf_evlist *evlist __maybe_unused)
1047 {
1048         if (!term->err_val) {
1049                 pr_debug("Config value not set\n");
1050                 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1051         }
1052
1053         if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) {
1054                 pr_debug("ERROR: wrong value type for 'value'\n");
1055                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1056         }
1057
1058         return __bpf_map__config_value(map, term);
1059 }
1060
1061 static int
1062 __bpf_map__config_event(struct bpf_map *map,
1063                         struct parse_events_term *term,
1064                         struct perf_evlist *evlist)
1065 {
1066         struct perf_evsel *evsel;
1067         const struct bpf_map_def *def;
1068         struct bpf_map_op *op;
1069         const char *map_name = bpf_map__name(map);
1070
1071         evsel = perf_evlist__find_evsel_by_str(evlist, term->val.str);
1072         if (!evsel) {
1073                 pr_debug("Event (for '%s') '%s' doesn't exist\n",
1074                          map_name, term->val.str);
1075                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT;
1076         }
1077
1078         def = bpf_map__def(map);
1079         if (IS_ERR(def)) {
1080                 pr_debug("Unable to get map definition from '%s'\n",
1081                          map_name);
1082                 return PTR_ERR(def);
1083         }
1084
1085         /*
1086          * No need to check key_size and value_size:
1087          * kernel has already checked them.
1088          */
1089         if (def->type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
1090                 pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
1091                          map_name);
1092                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1093         }
1094
1095         op = bpf_map__add_newop(map, term);
1096         if (IS_ERR(op))
1097                 return PTR_ERR(op);
1098         op->op_type = BPF_MAP_OP_SET_EVSEL;
1099         op->v.evsel = evsel;
1100         return 0;
1101 }
1102
1103 static int
1104 bpf_map__config_event(struct bpf_map *map,
1105                       struct parse_events_term *term,
1106                       struct perf_evlist *evlist)
1107 {
1108         if (!term->err_val) {
1109                 pr_debug("Config value not set\n");
1110                 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1111         }
1112
1113         if (term->type_val != PARSE_EVENTS__TERM_TYPE_STR) {
1114                 pr_debug("ERROR: wrong value type for 'event'\n");
1115                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1116         }
1117
1118         return __bpf_map__config_event(map, term, evlist);
1119 }
1120
1121 struct bpf_obj_config__map_func {
1122         const char *config_opt;
1123         int (*config_func)(struct bpf_map *, struct parse_events_term *,
1124                            struct perf_evlist *);
1125 };
1126
1127 struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = {
1128         {"value", bpf_map__config_value},
1129         {"event", bpf_map__config_event},
1130 };
1131
1132 static int
1133 config_map_indices_range_check(struct parse_events_term *term,
1134                                struct bpf_map *map,
1135                                const char *map_name)
1136 {
1137         struct parse_events_array *array = &term->array;
1138         const struct bpf_map_def *def;
1139         unsigned int i;
1140
1141         if (!array->nr_ranges)
1142                 return 0;
1143         if (!array->ranges) {
1144                 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n",
1145                          map_name, (int)array->nr_ranges);
1146                 return -BPF_LOADER_ERRNO__INTERNAL;
1147         }
1148
1149         def = bpf_map__def(map);
1150         if (IS_ERR(def)) {
1151                 pr_debug("ERROR: Unable to get map definition from '%s'\n",
1152                          map_name);
1153                 return -BPF_LOADER_ERRNO__INTERNAL;
1154         }
1155
1156         for (i = 0; i < array->nr_ranges; i++) {
1157                 unsigned int start = array->ranges[i].start;
1158                 size_t length = array->ranges[i].length;
1159                 unsigned int idx = start + length - 1;
1160
1161                 if (idx >= def->max_entries) {
1162                         pr_debug("ERROR: index %d too large\n", idx);
1163                         return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG;
1164                 }
1165         }
1166         return 0;
1167 }
1168
1169 static int
1170 bpf__obj_config_map(struct bpf_object *obj,
1171                     struct parse_events_term *term,
1172                     struct perf_evlist *evlist,
1173                     int *key_scan_pos)
1174 {
1175         /* key is "map:<mapname>.<config opt>" */
1176         char *map_name = strdup(term->config + sizeof("map:") - 1);
1177         struct bpf_map *map;
1178         int err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1179         char *map_opt;
1180         size_t i;
1181
1182         if (!map_name)
1183                 return -ENOMEM;
1184
1185         map_opt = strchr(map_name, '.');
1186         if (!map_opt) {
1187                 pr_debug("ERROR: Invalid map config: %s\n", map_name);
1188                 goto out;
1189         }
1190
1191         *map_opt++ = '\0';
1192         if (*map_opt == '\0') {
1193                 pr_debug("ERROR: Invalid map option: %s\n", term->config);
1194                 goto out;
1195         }
1196
1197         map = bpf_object__find_map_by_name(obj, map_name);
1198         if (!map) {
1199                 pr_debug("ERROR: Map %s doesn't exist\n", map_name);
1200                 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST;
1201                 goto out;
1202         }
1203
1204         *key_scan_pos += strlen(map_opt);
1205         err = config_map_indices_range_check(term, map, map_name);
1206         if (err)
1207                 goto out;
1208         *key_scan_pos -= strlen(map_opt);
1209
1210         for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) {
1211                 struct bpf_obj_config__map_func *func =
1212                                 &bpf_obj_config__map_funcs[i];
1213
1214                 if (strcmp(map_opt, func->config_opt) == 0) {
1215                         err = func->config_func(map, term, evlist);
1216                         goto out;
1217                 }
1218         }
1219
1220         pr_debug("ERROR: Invalid map config option '%s'\n", map_opt);
1221         err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT;
1222 out:
1223         free(map_name);
1224         if (!err)
1225                 key_scan_pos += strlen(map_opt);
1226         return err;
1227 }
1228
1229 int bpf__config_obj(struct bpf_object *obj,
1230                     struct parse_events_term *term,
1231                     struct perf_evlist *evlist,
1232                     int *error_pos)
1233 {
1234         int key_scan_pos = 0;
1235         int err;
1236
1237         if (!obj || !term || !term->config)
1238                 return -EINVAL;
1239
1240         if (strstarts(term->config, "map:")) {
1241                 key_scan_pos = sizeof("map:") - 1;
1242                 err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos);
1243                 goto out;
1244         }
1245         err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1246 out:
1247         if (error_pos)
1248                 *error_pos = key_scan_pos;
1249         return err;
1250
1251 }
1252
1253 typedef int (*map_config_func_t)(const char *name, int map_fd,
1254                                  const struct bpf_map_def *pdef,
1255                                  struct bpf_map_op *op,
1256                                  void *pkey, void *arg);
1257
1258 static int
1259 foreach_key_array_all(map_config_func_t func,
1260                       void *arg, const char *name,
1261                       int map_fd, const struct bpf_map_def *pdef,
1262                       struct bpf_map_op *op)
1263 {
1264         unsigned int i;
1265         int err;
1266
1267         for (i = 0; i < pdef->max_entries; i++) {
1268                 err = func(name, map_fd, pdef, op, &i, arg);
1269                 if (err) {
1270                         pr_debug("ERROR: failed to insert value to %s[%u]\n",
1271                                  name, i);
1272                         return err;
1273                 }
1274         }
1275         return 0;
1276 }
1277
1278 static int
1279 foreach_key_array_ranges(map_config_func_t func, void *arg,
1280                          const char *name, int map_fd,
1281                          const struct bpf_map_def *pdef,
1282                          struct bpf_map_op *op)
1283 {
1284         unsigned int i, j;
1285         int err;
1286
1287         for (i = 0; i < op->k.array.nr_ranges; i++) {
1288                 unsigned int start = op->k.array.ranges[i].start;
1289                 size_t length = op->k.array.ranges[i].length;
1290
1291                 for (j = 0; j < length; j++) {
1292                         unsigned int idx = start + j;
1293
1294                         err = func(name, map_fd, pdef, op, &idx, arg);
1295                         if (err) {
1296                                 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1297                                          name, idx);
1298                                 return err;
1299                         }
1300                 }
1301         }
1302         return 0;
1303 }
1304
1305 static int
1306 bpf_map_config_foreach_key(struct bpf_map *map,
1307                            map_config_func_t func,
1308                            void *arg)
1309 {
1310         int err, map_fd;
1311         struct bpf_map_op *op;
1312         const struct bpf_map_def *def;
1313         const char *name = bpf_map__name(map);
1314         struct bpf_map_priv *priv = bpf_map__priv(map);
1315
1316         if (IS_ERR(priv)) {
1317                 pr_debug("ERROR: failed to get private from map %s\n", name);
1318                 return -BPF_LOADER_ERRNO__INTERNAL;
1319         }
1320         if (!priv || list_empty(&priv->ops_list)) {
1321                 pr_debug("INFO: nothing to config for map %s\n", name);
1322                 return 0;
1323         }
1324
1325         def = bpf_map__def(map);
1326         if (IS_ERR(def)) {
1327                 pr_debug("ERROR: failed to get definition from map %s\n", name);
1328                 return -BPF_LOADER_ERRNO__INTERNAL;
1329         }
1330         map_fd = bpf_map__fd(map);
1331         if (map_fd < 0) {
1332                 pr_debug("ERROR: failed to get fd from map %s\n", name);
1333                 return map_fd;
1334         }
1335
1336         list_for_each_entry(op, &priv->ops_list, list) {
1337                 switch (def->type) {
1338                 case BPF_MAP_TYPE_ARRAY:
1339                 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1340                         switch (op->key_type) {
1341                         case BPF_MAP_KEY_ALL:
1342                                 err = foreach_key_array_all(func, arg, name,
1343                                                             map_fd, def, op);
1344                                 break;
1345                         case BPF_MAP_KEY_RANGES:
1346                                 err = foreach_key_array_ranges(func, arg, name,
1347                                                                map_fd, def,
1348                                                                op);
1349                                 break;
1350                         default:
1351                                 pr_debug("ERROR: keytype for map '%s' invalid\n",
1352                                          name);
1353                                 return -BPF_LOADER_ERRNO__INTERNAL;
1354                         }
1355                         if (err)
1356                                 return err;
1357                         break;
1358                 default:
1359                         pr_debug("ERROR: type of '%s' incorrect\n", name);
1360                         return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1361                 }
1362         }
1363
1364         return 0;
1365 }
1366
1367 static int
1368 apply_config_value_for_key(int map_fd, void *pkey,
1369                            size_t val_size, u64 val)
1370 {
1371         int err = 0;
1372
1373         switch (val_size) {
1374         case 1: {
1375                 u8 _val = (u8)(val);
1376                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1377                 break;
1378         }
1379         case 2: {
1380                 u16 _val = (u16)(val);
1381                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1382                 break;
1383         }
1384         case 4: {
1385                 u32 _val = (u32)(val);
1386                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1387                 break;
1388         }
1389         case 8: {
1390                 err = bpf_map_update_elem(map_fd, pkey, &val, BPF_ANY);
1391                 break;
1392         }
1393         default:
1394                 pr_debug("ERROR: invalid value size\n");
1395                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1396         }
1397         if (err && errno)
1398                 err = -errno;
1399         return err;
1400 }
1401
1402 static int
1403 apply_config_evsel_for_key(const char *name, int map_fd, void *pkey,
1404                            struct perf_evsel *evsel)
1405 {
1406         struct xyarray *xy = evsel->fd;
1407         struct perf_event_attr *attr;
1408         unsigned int key, events;
1409         bool check_pass = false;
1410         int *evt_fd;
1411         int err;
1412
1413         if (!xy) {
1414                 pr_debug("ERROR: evsel not ready for map %s\n", name);
1415                 return -BPF_LOADER_ERRNO__INTERNAL;
1416         }
1417
1418         if (xy->row_size / xy->entry_size != 1) {
1419                 pr_debug("ERROR: Dimension of target event is incorrect for map %s\n",
1420                          name);
1421                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM;
1422         }
1423
1424         attr = &evsel->attr;
1425         if (attr->inherit) {
1426                 pr_debug("ERROR: Can't put inherit event into map %s\n", name);
1427                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH;
1428         }
1429
1430         if (perf_evsel__is_bpf_output(evsel))
1431                 check_pass = true;
1432         if (attr->type == PERF_TYPE_RAW)
1433                 check_pass = true;
1434         if (attr->type == PERF_TYPE_HARDWARE)
1435                 check_pass = true;
1436         if (!check_pass) {
1437                 pr_debug("ERROR: Event type is wrong for map %s\n", name);
1438                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE;
1439         }
1440
1441         events = xy->entries / (xy->row_size / xy->entry_size);
1442         key = *((unsigned int *)pkey);
1443         if (key >= events) {
1444                 pr_debug("ERROR: there is no event %d for map %s\n",
1445                          key, name);
1446                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE;
1447         }
1448         evt_fd = xyarray__entry(xy, key, 0);
1449         err = bpf_map_update_elem(map_fd, pkey, evt_fd, BPF_ANY);
1450         if (err && errno)
1451                 err = -errno;
1452         return err;
1453 }
1454
1455 static int
1456 apply_obj_config_map_for_key(const char *name, int map_fd,
1457                              const struct bpf_map_def *pdef,
1458                              struct bpf_map_op *op,
1459                              void *pkey, void *arg __maybe_unused)
1460 {
1461         int err;
1462
1463         switch (op->op_type) {
1464         case BPF_MAP_OP_SET_VALUE:
1465                 err = apply_config_value_for_key(map_fd, pkey,
1466                                                  pdef->value_size,
1467                                                  op->v.value);
1468                 break;
1469         case BPF_MAP_OP_SET_EVSEL:
1470                 err = apply_config_evsel_for_key(name, map_fd, pkey,
1471                                                  op->v.evsel);
1472                 break;
1473         default:
1474                 pr_debug("ERROR: unknown value type for '%s'\n", name);
1475                 err = -BPF_LOADER_ERRNO__INTERNAL;
1476         }
1477         return err;
1478 }
1479
1480 static int
1481 apply_obj_config_map(struct bpf_map *map)
1482 {
1483         return bpf_map_config_foreach_key(map,
1484                                           apply_obj_config_map_for_key,
1485                                           NULL);
1486 }
1487
1488 static int
1489 apply_obj_config_object(struct bpf_object *obj)
1490 {
1491         struct bpf_map *map;
1492         int err;
1493
1494         bpf_object__for_each_map(map, obj) {
1495                 err = apply_obj_config_map(map);
1496                 if (err)
1497                         return err;
1498         }
1499         return 0;
1500 }
1501
1502 int bpf__apply_obj_config(void)
1503 {
1504         struct bpf_object *obj, *tmp;
1505         int err;
1506
1507         bpf_object__for_each_safe(obj, tmp) {
1508                 err = apply_obj_config_object(obj);
1509                 if (err)
1510                         return err;
1511         }
1512
1513         return 0;
1514 }
1515
1516 #define bpf__for_each_map(pos, obj, objtmp)     \
1517         bpf_object__for_each_safe(obj, objtmp)  \
1518                 bpf_object__for_each_map(pos, obj)
1519
1520 #define bpf__for_each_map_named(pos, obj, objtmp, name) \
1521         bpf__for_each_map(pos, obj, objtmp)             \
1522                 if (bpf_map__name(pos) &&               \
1523                         (strcmp(name,                   \
1524                                 bpf_map__name(pos)) == 0))
1525
1526 struct perf_evsel *bpf__setup_output_event(struct perf_evlist *evlist, const char *name)
1527 {
1528         struct bpf_map_priv *tmpl_priv = NULL;
1529         struct bpf_object *obj, *tmp;
1530         struct perf_evsel *evsel = NULL;
1531         struct bpf_map *map;
1532         int err;
1533         bool need_init = false;
1534
1535         bpf__for_each_map_named(map, obj, tmp, name) {
1536                 struct bpf_map_priv *priv = bpf_map__priv(map);
1537
1538                 if (IS_ERR(priv))
1539                         return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL);
1540
1541                 /*
1542                  * No need to check map type: type should have been
1543                  * verified by kernel.
1544                  */
1545                 if (!need_init && !priv)
1546                         need_init = !priv;
1547                 if (!tmpl_priv && priv)
1548                         tmpl_priv = priv;
1549         }
1550
1551         if (!need_init)
1552                 return NULL;
1553
1554         if (!tmpl_priv) {
1555                 char *event_definition = NULL;
1556
1557                 if (asprintf(&event_definition, "bpf-output/no-inherit=1,name=%s/", name) < 0)
1558                         return ERR_PTR(-ENOMEM);
1559
1560                 err = parse_events(evlist, event_definition, NULL);
1561                 free(event_definition);
1562
1563                 if (err) {
1564                         pr_debug("ERROR: failed to create the \"%s\" bpf-output event\n", name);
1565                         return ERR_PTR(-err);
1566                 }
1567
1568                 evsel = perf_evlist__last(evlist);
1569         }
1570
1571         bpf__for_each_map_named(map, obj, tmp, name) {
1572                 struct bpf_map_priv *priv = bpf_map__priv(map);
1573
1574                 if (IS_ERR(priv))
1575                         return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL);
1576                 if (priv)
1577                         continue;
1578
1579                 if (tmpl_priv) {
1580                         priv = bpf_map_priv__clone(tmpl_priv);
1581                         if (!priv)
1582                                 return ERR_PTR(-ENOMEM);
1583
1584                         err = bpf_map__set_priv(map, priv, bpf_map_priv__clear);
1585                         if (err) {
1586                                 bpf_map_priv__clear(map, priv);
1587                                 return ERR_PTR(err);
1588                         }
1589                 } else if (evsel) {
1590                         struct bpf_map_op *op;
1591
1592                         op = bpf_map__add_newop(map, NULL);
1593                         if (IS_ERR(op))
1594                                 return ERR_CAST(op);
1595                         op->op_type = BPF_MAP_OP_SET_EVSEL;
1596                         op->v.evsel = evsel;
1597                 }
1598         }
1599
1600         return evsel;
1601 }
1602
1603 int bpf__setup_stdout(struct perf_evlist *evlist)
1604 {
1605         struct perf_evsel *evsel = bpf__setup_output_event(evlist, "__bpf_stdout__");
1606         return PTR_ERR_OR_ZERO(evsel);
1607 }
1608
1609 #define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
1610 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1611 #define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1612
1613 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
1614         [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
1615         [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
1616         [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
1617         [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
1618         [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
1619         [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string",
1620         [ERRCODE_OFFSET(PROLOGUE)]      = "Failed to generate prologue",
1621         [ERRCODE_OFFSET(PROLOGUE2BIG)]  = "Prologue too big for program",
1622         [ERRCODE_OFFSET(PROLOGUEOOB)]   = "Offset out of bound for prologue",
1623         [ERRCODE_OFFSET(OBJCONF_OPT)]   = "Invalid object config option",
1624         [ERRCODE_OFFSET(OBJCONF_CONF)]  = "Config value not set (missing '=')",
1625         [ERRCODE_OFFSET(OBJCONF_MAP_OPT)]       = "Invalid object map config option",
1626         [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)]  = "Target map doesn't exist",
1627         [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)]     = "Incorrect value type for map",
1628         [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)]      = "Incorrect map type",
1629         [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)]   = "Incorrect map key size",
1630         [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size",
1631         [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT)]     = "Event not found for map setting",
1632         [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE)]   = "Invalid map size for event setting",
1633         [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM)]    = "Event dimension too large",
1634         [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH)]    = "Doesn't support inherit event",
1635         [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE)]   = "Wrong event type for map",
1636         [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG)]   = "Index too large",
1637 };
1638
1639 static int
1640 bpf_loader_strerror(int err, char *buf, size_t size)
1641 {
1642         char sbuf[STRERR_BUFSIZE];
1643         const char *msg;
1644
1645         if (!buf || !size)
1646                 return -1;
1647
1648         err = err > 0 ? err : -err;
1649
1650         if (err >= __LIBBPF_ERRNO__START)
1651                 return libbpf_strerror(err, buf, size);
1652
1653         if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
1654                 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
1655                 snprintf(buf, size, "%s", msg);
1656                 buf[size - 1] = '\0';
1657                 return 0;
1658         }
1659
1660         if (err >= __BPF_LOADER_ERRNO__END)
1661                 snprintf(buf, size, "Unknown bpf loader error %d", err);
1662         else
1663                 snprintf(buf, size, "%s",
1664                          str_error_r(err, sbuf, sizeof(sbuf)));
1665
1666         buf[size - 1] = '\0';
1667         return -1;
1668 }
1669
1670 #define bpf__strerror_head(err, buf, size) \
1671         char sbuf[STRERR_BUFSIZE], *emsg;\
1672         if (!size)\
1673                 return 0;\
1674         if (err < 0)\
1675                 err = -err;\
1676         bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1677         emsg = sbuf;\
1678         switch (err) {\
1679         default:\
1680                 scnprintf(buf, size, "%s", emsg);\
1681                 break;
1682
1683 #define bpf__strerror_entry(val, fmt...)\
1684         case val: {\
1685                 scnprintf(buf, size, fmt);\
1686                 break;\
1687         }
1688
1689 #define bpf__strerror_end(buf, size)\
1690         }\
1691         buf[size - 1] = '\0';
1692
1693 int bpf__strerror_prepare_load(const char *filename, bool source,
1694                                int err, char *buf, size_t size)
1695 {
1696         size_t n;
1697         int ret;
1698
1699         n = snprintf(buf, size, "Failed to load %s%s: ",
1700                          filename, source ? " from source" : "");
1701         if (n >= size) {
1702                 buf[size - 1] = '\0';
1703                 return 0;
1704         }
1705         buf += n;
1706         size -= n;
1707
1708         ret = bpf_loader_strerror(err, buf, size);
1709         buf[size - 1] = '\0';
1710         return ret;
1711 }
1712
1713 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
1714                         int err, char *buf, size_t size)
1715 {
1716         bpf__strerror_head(err, buf, size);
1717         case BPF_LOADER_ERRNO__PROGCONF_TERM: {
1718                 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
1719                 break;
1720         }
1721         bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1722         bpf__strerror_entry(EACCES, "You need to be root");
1723         bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1724         bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
1725         bpf__strerror_end(buf, size);
1726         return 0;
1727 }
1728
1729 int bpf__strerror_load(struct bpf_object *obj,
1730                        int err, char *buf, size_t size)
1731 {
1732         bpf__strerror_head(err, buf, size);
1733         case LIBBPF_ERRNO__KVER: {
1734                 unsigned int obj_kver = bpf_object__kversion(obj);
1735                 unsigned int real_kver;
1736
1737                 if (fetch_kernel_version(&real_kver, NULL, 0)) {
1738                         scnprintf(buf, size, "Unable to fetch kernel version");
1739                         break;
1740                 }
1741
1742                 if (obj_kver != real_kver) {
1743                         scnprintf(buf, size,
1744                                   "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
1745                                   KVER_PARAM(obj_kver),
1746                                   KVER_PARAM(real_kver));
1747                         break;
1748                 }
1749
1750                 scnprintf(buf, size, "Failed to load program for unknown reason");
1751                 break;
1752         }
1753         bpf__strerror_end(buf, size);
1754         return 0;
1755 }
1756
1757 int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused,
1758                              struct parse_events_term *term __maybe_unused,
1759                              struct perf_evlist *evlist __maybe_unused,
1760                              int *error_pos __maybe_unused, int err,
1761                              char *buf, size_t size)
1762 {
1763         bpf__strerror_head(err, buf, size);
1764         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE,
1765                             "Can't use this config term with this map type");
1766         bpf__strerror_end(buf, size);
1767         return 0;
1768 }
1769
1770 int bpf__strerror_apply_obj_config(int err, char *buf, size_t size)
1771 {
1772         bpf__strerror_head(err, buf, size);
1773         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM,
1774                             "Cannot set event to BPF map in multi-thread tracing");
1775         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH,
1776                             "%s (Hint: use -i to turn off inherit)", emsg);
1777         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE,
1778                             "Can only put raw, hardware and BPF output event into a BPF map");
1779         bpf__strerror_end(buf, size);
1780         return 0;
1781 }
1782
1783 int bpf__strerror_setup_output_event(struct perf_evlist *evlist __maybe_unused,
1784                                      int err, char *buf, size_t size)
1785 {
1786         bpf__strerror_head(err, buf, size);
1787         bpf__strerror_end(buf, size);
1788         return 0;
1789 }