kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18 #include "trace_dynevent.h"
  19
  20 #define SYNTH_SYSTEM            "synthetic"
  21 #define SYNTH_FIELDS_MAX        16
  22
  23 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  24
  25 #define ERRORS                                                          \
  26         C(NONE,                 "No error"),                            \
  27         C(DUPLICATE_VAR,        "Variable already defined"),            \
  28         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  29         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  30         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  31         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  32         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  33         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  34         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  35         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  36         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  37         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  38         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  39         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  40         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  41         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  42         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  43         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  44         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  45         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  46         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  47         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  48         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  49         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  50         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  51         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  52         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  53         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  54         C(INVALID_PARAM,        "Invalid action param"),                \
  55         C(ACTION_NOT_FOUND,     "No action found"),                     \
  56         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  57         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  58         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  59         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  60         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  61         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  62         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  63         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  64         C(FIELD_NOT_FOUND,      "Couldn't find field"),
  65
  66 #undef C
  67 #define C(a, b)         HIST_ERR_##a
  68
  69 enum { ERRORS };
  70
  71 #undef C
  72 #define C(a, b)         b
  73
  74 static const char *err_text[] = { ERRORS };
  75
  76 struct hist_field;
  77
  78 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  79                                 struct tracing_map_elt *elt,
  80                                 struct ring_buffer_event *rbe,
  81                                 void *event);
  82
  83 #define HIST_FIELD_OPERANDS_MAX 2
  84 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  85 #define HIST_ACTIONS_MAX        8
  86
  87 enum field_op_id {
  88         FIELD_OP_NONE,
  89         FIELD_OP_PLUS,
  90         FIELD_OP_MINUS,
  91         FIELD_OP_UNARY_MINUS,
  92 };
  93
  94 /*
  95  * A hist_var (histogram variable) contains variable information for
  96  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
  97  * flag set.  A hist_var has a variable name e.g. ts0, and is
  98  * associated with a given histogram trigger, as specified by
  99  * hist_data.  The hist_var idx is the unique index assigned to the
 100  * variable by the hist trigger's tracing_map.  The idx is what is
 101  * used to set a variable's value and, by a variable reference, to
 102  * retrieve it.
 103  */
 104 struct hist_var {
 105         char                            *name;
 106         struct hist_trigger_data        *hist_data;
 107         unsigned int                    idx;
 108 };
 109
 110 struct hist_field {
 111         struct ftrace_event_field       *field;
 112         unsigned long                   flags;
 113         hist_field_fn_t                 fn;
 114         unsigned int                    size;
 115         unsigned int                    offset;
 116         unsigned int                    is_signed;
 117         const char                      *type;
 118         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 119         struct hist_trigger_data        *hist_data;
 120
 121         /*
 122          * Variable fields contain variable-specific info in var.
 123          */
 124         struct hist_var                 var;
 125         enum field_op_id                operator;
 126         char                            *system;
 127         char                            *event_name;
 128
 129         /*
 130          * The name field is used for EXPR and VAR_REF fields.  VAR
 131          * fields contain the variable name in var.name.
 132          */
 133         char                            *name;
 134
 135         /*
 136          * When a histogram trigger is hit, if it has any references
 137          * to variables, the values of those variables are collected
 138          * into a var_ref_vals array by resolve_var_refs().  The
 139          * current value of each variable is read from the tracing_map
 140          * using the hist field's hist_var.idx and entered into the
 141          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 142          */
 143         unsigned int                    var_ref_idx;
 144         bool                            read_once;
 145 };
 146
 147 static u64 hist_field_none(struct hist_field *field,
 148                            struct tracing_map_elt *elt,
 149                            struct ring_buffer_event *rbe,
 150                            void *event)
 151 {
 152         return 0;
 153 }
 154
 155 static u64 hist_field_counter(struct hist_field *field,
 156                               struct tracing_map_elt *elt,
 157                               struct ring_buffer_event *rbe,
 158                               void *event)
 159 {
 160         return 1;
 161 }
 162
 163 static u64 hist_field_string(struct hist_field *hist_field,
 164                              struct tracing_map_elt *elt,
 165                              struct ring_buffer_event *rbe,
 166                              void *event)
 167 {
 168         char *addr = (char *)(event + hist_field->field->offset);
 169
 170         return (u64)(unsigned long)addr;
 171 }
 172
 173 static u64 hist_field_dynstring(struct hist_field *hist_field,
 174                                 struct tracing_map_elt *elt,
 175                                 struct ring_buffer_event *rbe,
 176                                 void *event)
 177 {
 178         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 179         int str_loc = str_item & 0xffff;
 180         char *addr = (char *)(event + str_loc);
 181
 182         return (u64)(unsigned long)addr;
 183 }
 184
 185 static u64 hist_field_pstring(struct hist_field *hist_field,
 186                               struct tracing_map_elt *elt,
 187                               struct ring_buffer_event *rbe,
 188                               void *event)
 189 {
 190         char **addr = (char **)(event + hist_field->field->offset);
 191
 192         return (u64)(unsigned long)*addr;
 193 }
 194
 195 static u64 hist_field_log2(struct hist_field *hist_field,
 196                            struct tracing_map_elt *elt,
 197                            struct ring_buffer_event *rbe,
 198                            void *event)
 199 {
 200         struct hist_field *operand = hist_field->operands[0];
 201
 202         u64 val = operand->fn(operand, elt, rbe, event);
 203
 204         return (u64) ilog2(roundup_pow_of_two(val));
 205 }
 206
 207 static u64 hist_field_plus(struct hist_field *hist_field,
 208                            struct tracing_map_elt *elt,
 209                            struct ring_buffer_event *rbe,
 210                            void *event)
 211 {
 212         struct hist_field *operand1 = hist_field->operands[0];
 213         struct hist_field *operand2 = hist_field->operands[1];
 214
 215         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 216         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 217
 218         return val1 + val2;
 219 }
 220
 221 static u64 hist_field_minus(struct hist_field *hist_field,
 222                             struct tracing_map_elt *elt,
 223                             struct ring_buffer_event *rbe,
 224                             void *event)
 225 {
 226         struct hist_field *operand1 = hist_field->operands[0];
 227         struct hist_field *operand2 = hist_field->operands[1];
 228
 229         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 230         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 231
 232         return val1 - val2;
 233 }
 234
 235 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 236                                   struct tracing_map_elt *elt,
 237                                   struct ring_buffer_event *rbe,
 238                                   void *event)
 239 {
 240         struct hist_field *operand = hist_field->operands[0];
 241
 242         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 243         u64 val = (u64)-sval;
 244
 245         return val;
 246 }
 247
 248 #define DEFINE_HIST_FIELD_FN(type)                                      \
 249         static u64 hist_field_##type(struct hist_field *hist_field,     \
 250                                      struct tracing_map_elt *elt,       \
 251                                      struct ring_buffer_event *rbe,     \
 252                                      void *event)                       \
 253 {                                                                       \
 254         type *addr = (type *)(event + hist_field->field->offset);       \
 255                                                                         \
 256         return (u64)(unsigned long)*addr;                               \
 257 }
 258
 259 DEFINE_HIST_FIELD_FN(s64);
 260 DEFINE_HIST_FIELD_FN(u64);
 261 DEFINE_HIST_FIELD_FN(s32);
 262 DEFINE_HIST_FIELD_FN(u32);
 263 DEFINE_HIST_FIELD_FN(s16);
 264 DEFINE_HIST_FIELD_FN(u16);
 265 DEFINE_HIST_FIELD_FN(s8);
 266 DEFINE_HIST_FIELD_FN(u8);
 267
 268 #define for_each_hist_field(i, hist_data)       \
 269         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 270
 271 #define for_each_hist_val_field(i, hist_data)   \
 272         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 273
 274 #define for_each_hist_key_field(i, hist_data)   \
 275         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 276
 277 #define HIST_STACKTRACE_DEPTH   16
 278 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 279 #define HIST_STACKTRACE_SKIP    5
 280
 281 #define HITCOUNT_IDX            0
 282 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 283
 284 enum hist_field_flags {
 285         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 286         HIST_FIELD_FL_KEY               = 1 << 1,
 287         HIST_FIELD_FL_STRING            = 1 << 2,
 288         HIST_FIELD_FL_HEX               = 1 << 3,
 289         HIST_FIELD_FL_SYM               = 1 << 4,
 290         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 291         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 292         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 293         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 294         HIST_FIELD_FL_LOG2              = 1 << 9,
 295         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 296         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 297         HIST_FIELD_FL_VAR               = 1 << 12,
 298         HIST_FIELD_FL_EXPR              = 1 << 13,
 299         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 300         HIST_FIELD_FL_CPU               = 1 << 15,
 301         HIST_FIELD_FL_ALIAS             = 1 << 16,
 302 };
 303
 304 struct var_defs {
 305         unsigned int    n_vars;
 306         char            *name[TRACING_MAP_VARS_MAX];
 307         char            *expr[TRACING_MAP_VARS_MAX];
 308 };
 309
 310 struct hist_trigger_attrs {
 311         char            *keys_str;
 312         char            *vals_str;
 313         char            *sort_key_str;
 314         char            *name;
 315         char            *clock;
 316         bool            pause;
 317         bool            cont;
 318         bool            clear;
 319         bool            ts_in_usecs;
 320         unsigned int    map_bits;
 321
 322         char            *assignment_str[TRACING_MAP_VARS_MAX];
 323         unsigned int    n_assignments;
 324
 325         char            *action_str[HIST_ACTIONS_MAX];
 326         unsigned int    n_actions;
 327
 328         struct var_defs var_defs;
 329 };
 330
 331 struct field_var {
 332         struct hist_field       *var;
 333         struct hist_field       *val;
 334 };
 335
 336 struct field_var_hist {
 337         struct hist_trigger_data        *hist_data;
 338         char                            *cmd;
 339 };
 340
 341 struct hist_trigger_data {
 342         struct hist_field               *fields[HIST_FIELDS_MAX];
 343         unsigned int                    n_vals;
 344         unsigned int                    n_keys;
 345         unsigned int                    n_fields;
 346         unsigned int                    n_vars;
 347         unsigned int                    key_size;
 348         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 349         unsigned int                    n_sort_keys;
 350         struct trace_event_file         *event_file;
 351         struct hist_trigger_attrs       *attrs;
 352         struct tracing_map              *map;
 353         bool                            enable_timestamps;
 354         bool                            remove;
 355         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 356         unsigned int                    n_var_refs;
 357
 358         struct action_data              *actions[HIST_ACTIONS_MAX];
 359         unsigned int                    n_actions;
 360
 361         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 362         unsigned int                    n_field_vars;
 363         unsigned int                    n_field_var_str;
 364         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 365         unsigned int                    n_field_var_hists;
 366
 367         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 368         unsigned int                    n_save_vars;
 369         unsigned int                    n_save_var_str;
 370 };
 371
 372 static int synth_event_create(int argc, const char **argv);
 373 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 374 static int synth_event_release(struct dyn_event *ev);
 375 static bool synth_event_is_busy(struct dyn_event *ev);
 376 static bool synth_event_match(const char *system, const char *event,
 377                               struct dyn_event *ev);
 378
 379 static struct dyn_event_operations synth_event_ops = {
 380         .create = synth_event_create,
 381         .show = synth_event_show,
 382         .is_busy = synth_event_is_busy,
 383         .free = synth_event_release,
 384         .match = synth_event_match,
 385 };
 386
 387 struct synth_field {
 388         char *type;
 389         char *name;
 390         size_t size;
 391         bool is_signed;
 392         bool is_string;
 393 };
 394
 395 struct synth_event {
 396         struct dyn_event                        devent;
 397         int                                     ref;
 398         char                                    *name;
 399         struct synth_field                      **fields;
 400         unsigned int                            n_fields;
 401         unsigned int                            n_u64;
 402         struct trace_event_class                class;
 403         struct trace_event_call                 call;
 404         struct tracepoint                       *tp;
 405 };
 406
 407 static bool is_synth_event(struct dyn_event *ev)
 408 {
 409         return ev->ops == &synth_event_ops;
 410 }
 411
 412 static struct synth_event *to_synth_event(struct dyn_event *ev)
 413 {
 414         return container_of(ev, struct synth_event, devent);
 415 }
 416
 417 static bool synth_event_is_busy(struct dyn_event *ev)
 418 {
 419         struct synth_event *event = to_synth_event(ev);
 420
 421         return event->ref != 0;
 422 }
 423
 424 static bool synth_event_match(const char *system, const char *event,
 425                               struct dyn_event *ev)
 426 {
 427         struct synth_event *sev = to_synth_event(ev);
 428
 429         return strcmp(sev->name, event) == 0 &&
 430                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 431 }
 432
 433 struct action_data;
 434
 435 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 436                              struct tracing_map_elt *elt, void *rec,
 437                              struct ring_buffer_event *rbe, void *key,
 438                              struct action_data *data, u64 *var_ref_vals);
 439
 440 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 441
 442 enum handler_id {
 443         HANDLER_ONMATCH = 1,
 444         HANDLER_ONMAX,
 445         HANDLER_ONCHANGE,
 446 };
 447
 448 enum action_id {
 449         ACTION_SAVE = 1,
 450         ACTION_TRACE,
 451         ACTION_SNAPSHOT,
 452 };
 453
 454 struct action_data {
 455         enum handler_id         handler;
 456         enum action_id          action;
 457         char                    *action_name;
 458         action_fn_t             fn;
 459
 460         unsigned int            n_params;
 461         char                    *params[SYNTH_FIELDS_MAX];
 462
 463         /*
 464          * When a histogram trigger is hit, the values of any
 465          * references to variables, including variables being passed
 466          * as parameters to synthetic events, are collected into a
 467          * var_ref_vals array.  This var_ref_idx is the index of the
 468          * first param in the array to be passed to the synthetic
 469          * event invocation.
 470          */
 471         unsigned int            var_ref_idx;
 472         struct synth_event      *synth_event;
 473         bool                    use_trace_keyword;
 474         char                    *synth_event_name;
 475
 476         union {
 477                 struct {
 478                         char                    *event;
 479                         char                    *event_system;
 480                 } match_data;
 481
 482                 struct {
 483                         /*
 484                          * var_str contains the $-unstripped variable
 485                          * name referenced by var_ref, and used when
 486                          * printing the action.  Because var_ref
 487                          * creation is deferred to create_actions(),
 488                          * we need a per-action way to save it until
 489                          * then, thus var_str.
 490                          */
 491                         char                    *var_str;
 492
 493                         /*
 494                          * var_ref refers to the variable being
 495                          * tracked e.g onmax($var).
 496                          */
 497                         struct hist_field       *var_ref;
 498
 499                         /*
 500                          * track_var contains the 'invisible' tracking
 501                          * variable created to keep the current
 502                          * e.g. max value.
 503                          */
 504                         struct hist_field       *track_var;
 505
 506                         check_track_val_fn_t    check_val;
 507                         action_fn_t             save_data;
 508                 } track_data;
 509         };
 510 };
 511
 512 struct track_data {
 513         u64                             track_val;
 514         bool                            updated;
 515
 516         unsigned int                    key_len;
 517         void                            *key;
 518         struct tracing_map_elt          elt;
 519
 520         struct action_data              *action_data;
 521         struct hist_trigger_data        *hist_data;
 522 };
 523
 524 struct hist_elt_data {
 525         char *comm;
 526         u64 *var_ref_vals;
 527         char *field_var_str[SYNTH_FIELDS_MAX];
 528 };
 529
 530 struct snapshot_context {
 531         struct tracing_map_elt  *elt;
 532         void                    *key;
 533 };
 534
 535 static void track_data_free(struct track_data *track_data)
 536 {
 537         struct hist_elt_data *elt_data;
 538
 539         if (!track_data)
 540                 return;
 541
 542         kfree(track_data->key);
 543
 544         elt_data = track_data->elt.private_data;
 545         if (elt_data) {
 546                 kfree(elt_data->comm);
 547                 kfree(elt_data);
 548         }
 549
 550         kfree(track_data);
 551 }
 552
 553 static struct track_data *track_data_alloc(unsigned int key_len,
 554                                            struct action_data *action_data,
 555                                            struct hist_trigger_data *hist_data)
 556 {
 557         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 558         struct hist_elt_data *elt_data;
 559
 560         if (!data)
 561                 return ERR_PTR(-ENOMEM);
 562
 563         data->key = kzalloc(key_len, GFP_KERNEL);
 564         if (!data->key) {
 565                 track_data_free(data);
 566                 return ERR_PTR(-ENOMEM);
 567         }
 568
 569         data->key_len = key_len;
 570         data->action_data = action_data;
 571         data->hist_data = hist_data;
 572
 573         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 574         if (!elt_data) {
 575                 track_data_free(data);
 576                 return ERR_PTR(-ENOMEM);
 577         }
 578         data->elt.private_data = elt_data;
 579
 580         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 581         if (!elt_data->comm) {
 582                 track_data_free(data);
 583                 return ERR_PTR(-ENOMEM);
 584         }
 585
 586         return data;
 587 }
 588
 589 static char last_cmd[MAX_FILTER_STR_VAL];
 590 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 591
 592 static int errpos(char *str)
 593 {
 594         return err_pos(last_cmd, str);
 595 }
 596
 597 static void last_cmd_set(struct trace_event_file *file, char *str)
 598 {
 599         const char *system = NULL, *name = NULL;
 600         struct trace_event_call *call;
 601
 602         if (!str)
 603                 return;
 604
 605         strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1);
 606
 607         if (file) {
 608                 call = file->event_call;
 609
 610                 system = call->class->system;
 611                 if (system) {
 612                         name = trace_event_name(call);
 613                         if (!name)
 614                                 system = NULL;
 615                 }
 616         }
 617
 618         if (system)
 619                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 620 }
 621
 622 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 623 {
 624         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 625                         err_type, err_pos);
 626 }
 627
 628 static void hist_err_clear(void)
 629 {
 630         last_cmd[0] = '\0';
 631         last_cmd_loc[0] = '\0';
 632 }
 633
 634 struct synth_trace_event {
 635         struct trace_entry      ent;
 636         u64                     fields[];
 637 };
 638
 639 static int synth_event_define_fields(struct trace_event_call *call)
 640 {
 641         struct synth_trace_event trace;
 642         int offset = offsetof(typeof(trace), fields);
 643         struct synth_event *event = call->data;
 644         unsigned int i, size, n_u64;
 645         char *name, *type;
 646         bool is_signed;
 647         int ret = 0;
 648
 649         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 650                 size = event->fields[i]->size;
 651                 is_signed = event->fields[i]->is_signed;
 652                 type = event->fields[i]->type;
 653                 name = event->fields[i]->name;
 654                 ret = trace_define_field(call, type, name, offset, size,
 655                                          is_signed, FILTER_OTHER);
 656                 if (ret)
 657                         break;
 658
 659                 if (event->fields[i]->is_string) {
 660                         offset += STR_VAR_LEN_MAX;
 661                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 662                 } else {
 663                         offset += sizeof(u64);
 664                         n_u64++;
 665                 }
 666         }
 667
 668         event->n_u64 = n_u64;
 669
 670         return ret;
 671 }
 672
 673 static bool synth_field_signed(char *type)
 674 {
 675         if (str_has_prefix(type, "u"))
 676                 return false;
 677         if (strcmp(type, "gfp_t") == 0)
 678                 return false;
 679
 680         return true;
 681 }
 682
 683 static int synth_field_is_string(char *type)
 684 {
 685         if (strstr(type, "char[") != NULL)
 686                 return true;
 687
 688         return false;
 689 }
 690
 691 static int synth_field_string_size(char *type)
 692 {
 693         char buf[4], *end, *start;
 694         unsigned int len;
 695         int size, err;
 696
 697         start = strstr(type, "char[");
 698         if (start == NULL)
 699                 return -EINVAL;
 700         start += sizeof("char[") - 1;
 701
 702         end = strchr(type, ']');
 703         if (!end || end < start)
 704                 return -EINVAL;
 705
 706         len = end - start;
 707         if (len > 3)
 708                 return -EINVAL;
 709
 710         strncpy(buf, start, len);
 711         buf[len] = '\0';
 712
 713         err = kstrtouint(buf, 0, &size);
 714         if (err)
 715                 return err;
 716
 717         if (size > STR_VAR_LEN_MAX)
 718                 return -EINVAL;
 719
 720         return size;
 721 }
 722
 723 static int synth_field_size(char *type)
 724 {
 725         int size = 0;
 726
 727         if (strcmp(type, "s64") == 0)
 728                 size = sizeof(s64);
 729         else if (strcmp(type, "u64") == 0)
 730                 size = sizeof(u64);
 731         else if (strcmp(type, "s32") == 0)
 732                 size = sizeof(s32);
 733         else if (strcmp(type, "u32") == 0)
 734                 size = sizeof(u32);
 735         else if (strcmp(type, "s16") == 0)
 736                 size = sizeof(s16);
 737         else if (strcmp(type, "u16") == 0)
 738                 size = sizeof(u16);
 739         else if (strcmp(type, "s8") == 0)
 740                 size = sizeof(s8);
 741         else if (strcmp(type, "u8") == 0)
 742                 size = sizeof(u8);
 743         else if (strcmp(type, "char") == 0)
 744                 size = sizeof(char);
 745         else if (strcmp(type, "unsigned char") == 0)
 746                 size = sizeof(unsigned char);
 747         else if (strcmp(type, "int") == 0)
 748                 size = sizeof(int);
 749         else if (strcmp(type, "unsigned int") == 0)
 750                 size = sizeof(unsigned int);
 751         else if (strcmp(type, "long") == 0)
 752                 size = sizeof(long);
 753         else if (strcmp(type, "unsigned long") == 0)
 754                 size = sizeof(unsigned long);
 755         else if (strcmp(type, "pid_t") == 0)
 756                 size = sizeof(pid_t);
 757         else if (synth_field_is_string(type))
 758                 size = synth_field_string_size(type);
 759
 760         return size;
 761 }
 762
 763 static const char *synth_field_fmt(char *type)
 764 {
 765         const char *fmt = "%llu";
 766
 767         if (strcmp(type, "s64") == 0)
 768                 fmt = "%lld";
 769         else if (strcmp(type, "u64") == 0)
 770                 fmt = "%llu";
 771         else if (strcmp(type, "s32") == 0)
 772                 fmt = "%d";
 773         else if (strcmp(type, "u32") == 0)
 774                 fmt = "%u";
 775         else if (strcmp(type, "s16") == 0)
 776                 fmt = "%d";
 777         else if (strcmp(type, "u16") == 0)
 778                 fmt = "%u";
 779         else if (strcmp(type, "s8") == 0)
 780                 fmt = "%d";
 781         else if (strcmp(type, "u8") == 0)
 782                 fmt = "%u";
 783         else if (strcmp(type, "char") == 0)
 784                 fmt = "%d";
 785         else if (strcmp(type, "unsigned char") == 0)
 786                 fmt = "%u";
 787         else if (strcmp(type, "int") == 0)
 788                 fmt = "%d";
 789         else if (strcmp(type, "unsigned int") == 0)
 790                 fmt = "%u";
 791         else if (strcmp(type, "long") == 0)
 792                 fmt = "%ld";
 793         else if (strcmp(type, "unsigned long") == 0)
 794                 fmt = "%lu";
 795         else if (strcmp(type, "pid_t") == 0)
 796                 fmt = "%d";
 797         else if (synth_field_is_string(type))
 798                 fmt = "%s";
 799
 800         return fmt;
 801 }
 802
 803 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 804                                            int flags,
 805                                            struct trace_event *event)
 806 {
 807         struct trace_array *tr = iter->tr;
 808         struct trace_seq *s = &iter->seq;
 809         struct synth_trace_event *entry;
 810         struct synth_event *se;
 811         unsigned int i, n_u64;
 812         char print_fmt[32];
 813         const char *fmt;
 814
 815         entry = (struct synth_trace_event *)iter->ent;
 816         se = container_of(event, struct synth_event, call.event);
 817
 818         trace_seq_printf(s, "%s: ", se->name);
 819
 820         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 821                 if (trace_seq_has_overflowed(s))
 822                         goto end;
 823
 824                 fmt = synth_field_fmt(se->fields[i]->type);
 825
 826                 /* parameter types */
 827                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 828                         trace_seq_printf(s, "%s ", fmt);
 829
 830                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 831
 832                 /* parameter values */
 833                 if (se->fields[i]->is_string) {
 834                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 835                                          (char *)&entry->fields[n_u64],
 836                                          i == se->n_fields - 1 ? "" : " ");
 837                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 838                 } else {
 839                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 840                                          entry->fields[n_u64],
 841                                          i == se->n_fields - 1 ? "" : " ");
 842                         n_u64++;
 843                 }
 844         }
 845 end:
 846         trace_seq_putc(s, '\n');
 847
 848         return trace_handle_return(s);
 849 }
 850
 851 static struct trace_event_functions synth_event_funcs = {
 852         .trace          = print_synth_event
 853 };
 854
 855 static notrace void trace_event_raw_event_synth(void *__data,
 856                                                 u64 *var_ref_vals,
 857                                                 unsigned int var_ref_idx)
 858 {
 859         struct trace_event_file *trace_file = __data;
 860         struct synth_trace_event *entry;
 861         struct trace_event_buffer fbuffer;
 862         struct ring_buffer *buffer;
 863         struct synth_event *event;
 864         unsigned int i, n_u64;
 865         int fields_size = 0;
 866
 867         event = trace_file->event_call->data;
 868
 869         if (trace_trigger_soft_disabled(trace_file))
 870                 return;
 871
 872         fields_size = event->n_u64 * sizeof(u64);
 873
 874         /*
 875          * Avoid ring buffer recursion detection, as this event
 876          * is being performed within another event.
 877          */
 878         buffer = trace_file->tr->trace_buffer.buffer;
 879         ring_buffer_nest_start(buffer);
 880
 881         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 882                                            sizeof(*entry) + fields_size);
 883         if (!entry)
 884                 goto out;
 885
 886         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 887                 if (event->fields[i]->is_string) {
 888                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 889                         char *str_field = (char *)&entry->fields[n_u64];
 890
 891                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 892                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 893                 } else {
 894                         entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 895                         n_u64++;
 896                 }
 897         }
 898
 899         trace_event_buffer_commit(&fbuffer);
 900 out:
 901         ring_buffer_nest_end(buffer);
 902 }
 903
 904 static void free_synth_event_print_fmt(struct trace_event_call *call)
 905 {
 906         if (call) {
 907                 kfree(call->print_fmt);
 908                 call->print_fmt = NULL;
 909         }
 910 }
 911
 912 static int __set_synth_event_print_fmt(struct synth_event *event,
 913                                        char *buf, int len)
 914 {
 915         const char *fmt;
 916         int pos = 0;
 917         int i;
 918
 919         /* When len=0, we just calculate the needed length */
 920 #define LEN_OR_ZERO (len ? len - pos : 0)
 921
 922         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 923         for (i = 0; i < event->n_fields; i++) {
 924                 fmt = synth_field_fmt(event->fields[i]->type);
 925                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 926                                 event->fields[i]->name, fmt,
 927                                 i == event->n_fields - 1 ? "" : ", ");
 928         }
 929         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 930
 931         for (i = 0; i < event->n_fields; i++) {
 932                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 933                                 ", REC->%s", event->fields[i]->name);
 934         }
 935
 936 #undef LEN_OR_ZERO
 937
 938         /* return the length of print_fmt */
 939         return pos;
 940 }
 941
 942 static int set_synth_event_print_fmt(struct trace_event_call *call)
 943 {
 944         struct synth_event *event = call->data;
 945         char *print_fmt;
 946         int len;
 947
 948         /* First: called with 0 length to calculate the needed length */
 949         len = __set_synth_event_print_fmt(event, NULL, 0);
 950
 951         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 952         if (!print_fmt)
 953                 return -ENOMEM;
 954
 955         /* Second: actually write the @print_fmt */
 956         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 957         call->print_fmt = print_fmt;
 958
 959         return 0;
 960 }
 961
 962 static void free_synth_field(struct synth_field *field)
 963 {
 964         kfree(field->type);
 965         kfree(field->name);
 966         kfree(field);
 967 }
 968
 969 static struct synth_field *parse_synth_field(int argc, const char **argv,
 970                                              int *consumed)
 971 {
 972         struct synth_field *field;
 973         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
 974         int len, ret = 0;
 975
 976         if (field_type[0] == ';')
 977                 field_type++;
 978
 979         if (!strcmp(field_type, "unsigned")) {
 980                 if (argc < 3)
 981                         return ERR_PTR(-EINVAL);
 982                 prefix = "unsigned ";
 983                 field_type = argv[1];
 984                 field_name = argv[2];
 985                 *consumed = 3;
 986         } else {
 987                 field_name = argv[1];
 988                 *consumed = 2;
 989         }
 990
 991         field = kzalloc(sizeof(*field), GFP_KERNEL);
 992         if (!field)
 993                 return ERR_PTR(-ENOMEM);
 994
 995         len = strlen(field_name);
 996         array = strchr(field_name, '[');
 997         if (array)
 998                 len -= strlen(array);
 999         else if (field_name[len - 1] == ';')
1000                 len--;
1001
1002         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1003         if (!field->name) {
1004                 ret = -ENOMEM;
1005                 goto free;
1006         }
1007
1008         if (field_type[0] == ';')
1009                 field_type++;
1010         len = strlen(field_type) + 1;
1011         if (array)
1012                 len += strlen(array);
1013         if (prefix)
1014                 len += strlen(prefix);
1015
1016         field->type = kzalloc(len, GFP_KERNEL);
1017         if (!field->type) {
1018                 ret = -ENOMEM;
1019                 goto free;
1020         }
1021         if (prefix)
1022                 strcat(field->type, prefix);
1023         strcat(field->type, field_type);
1024         if (array) {
1025                 strcat(field->type, array);
1026                 if (field->type[len - 1] == ';')
1027                         field->type[len - 1] = '\0';
1028         }
1029
1030         field->size = synth_field_size(field->type);
1031         if (!field->size) {
1032                 ret = -EINVAL;
1033                 goto free;
1034         }
1035
1036         if (synth_field_is_string(field->type))
1037                 field->is_string = true;
1038
1039         field->is_signed = synth_field_signed(field->type);
1040
1041  out:
1042         return field;
1043  free:
1044         free_synth_field(field);
1045         field = ERR_PTR(ret);
1046         goto out;
1047 }
1048
1049 static void free_synth_tracepoint(struct tracepoint *tp)
1050 {
1051         if (!tp)
1052                 return;
1053
1054         kfree(tp->name);
1055         kfree(tp);
1056 }
1057
1058 static struct tracepoint *alloc_synth_tracepoint(char *name)
1059 {
1060         struct tracepoint *tp;
1061
1062         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1063         if (!tp)
1064                 return ERR_PTR(-ENOMEM);
1065
1066         tp->name = kstrdup(name, GFP_KERNEL);
1067         if (!tp->name) {
1068                 kfree(tp);
1069                 return ERR_PTR(-ENOMEM);
1070         }
1071
1072         return tp;
1073 }
1074
1075 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1076                                     unsigned int var_ref_idx);
1077
1078 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1079                                unsigned int var_ref_idx)
1080 {
1081         struct tracepoint *tp = event->tp;
1082
1083         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1084                 struct tracepoint_func *probe_func_ptr;
1085                 synth_probe_func_t probe_func;
1086                 void *__data;
1087
1088                 if (!(cpu_online(raw_smp_processor_id())))
1089                         return;
1090
1091                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1092                 if (probe_func_ptr) {
1093                         do {
1094                                 probe_func = probe_func_ptr->func;
1095                                 __data = probe_func_ptr->data;
1096                                 probe_func(__data, var_ref_vals, var_ref_idx);
1097                         } while ((++probe_func_ptr)->func);
1098                 }
1099         }
1100 }
1101
1102 static struct synth_event *find_synth_event(const char *name)
1103 {
1104         struct dyn_event *pos;
1105         struct synth_event *event;
1106
1107         for_each_dyn_event(pos) {
1108                 if (!is_synth_event(pos))
1109                         continue;
1110                 event = to_synth_event(pos);
1111                 if (strcmp(event->name, name) == 0)
1112                         return event;
1113         }
1114
1115         return NULL;
1116 }
1117
1118 static int register_synth_event(struct synth_event *event)
1119 {
1120         struct trace_event_call *call = &event->call;
1121         int ret = 0;
1122
1123         event->call.class = &event->class;
1124         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1125         if (!event->class.system) {
1126                 ret = -ENOMEM;
1127                 goto out;
1128         }
1129
1130         event->tp = alloc_synth_tracepoint(event->name);
1131         if (IS_ERR(event->tp)) {
1132                 ret = PTR_ERR(event->tp);
1133                 event->tp = NULL;
1134                 goto out;
1135         }
1136
1137         INIT_LIST_HEAD(&call->class->fields);
1138         call->event.funcs = &synth_event_funcs;
1139         call->class->define_fields = synth_event_define_fields;
1140
1141         ret = register_trace_event(&call->event);
1142         if (!ret) {
1143                 ret = -ENODEV;
1144                 goto out;
1145         }
1146         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1147         call->class->reg = trace_event_reg;
1148         call->class->probe = trace_event_raw_event_synth;
1149         call->data = event;
1150         call->tp = event->tp;
1151
1152         ret = trace_add_event_call(call);
1153         if (ret) {
1154                 pr_warn("Failed to register synthetic event: %s\n",
1155                         trace_event_name(call));
1156                 goto err;
1157         }
1158
1159         ret = set_synth_event_print_fmt(call);
1160         if (ret < 0) {
1161                 trace_remove_event_call(call);
1162                 goto err;
1163         }
1164  out:
1165         return ret;
1166  err:
1167         unregister_trace_event(&call->event);
1168         goto out;
1169 }
1170
1171 static int unregister_synth_event(struct synth_event *event)
1172 {
1173         struct trace_event_call *call = &event->call;
1174         int ret;
1175
1176         ret = trace_remove_event_call(call);
1177
1178         return ret;
1179 }
1180
1181 static void free_synth_event(struct synth_event *event)
1182 {
1183         unsigned int i;
1184
1185         if (!event)
1186                 return;
1187
1188         for (i = 0; i < event->n_fields; i++)
1189                 free_synth_field(event->fields[i]);
1190
1191         kfree(event->fields);
1192         kfree(event->name);
1193         kfree(event->class.system);
1194         free_synth_tracepoint(event->tp);
1195         free_synth_event_print_fmt(&event->call);
1196         kfree(event);
1197 }
1198
1199 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1200                                              struct synth_field **fields)
1201 {
1202         struct synth_event *event;
1203         unsigned int i;
1204
1205         event = kzalloc(sizeof(*event), GFP_KERNEL);
1206         if (!event) {
1207                 event = ERR_PTR(-ENOMEM);
1208                 goto out;
1209         }
1210
1211         event->name = kstrdup(name, GFP_KERNEL);
1212         if (!event->name) {
1213                 kfree(event);
1214                 event = ERR_PTR(-ENOMEM);
1215                 goto out;
1216         }
1217
1218         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1219         if (!event->fields) {
1220                 free_synth_event(event);
1221                 event = ERR_PTR(-ENOMEM);
1222                 goto out;
1223         }
1224
1225         dyn_event_init(&event->devent, &synth_event_ops);
1226
1227         for (i = 0; i < n_fields; i++)
1228                 event->fields[i] = fields[i];
1229
1230         event->n_fields = n_fields;
1231  out:
1232         return event;
1233 }
1234
1235 static void action_trace(struct hist_trigger_data *hist_data,
1236                          struct tracing_map_elt *elt, void *rec,
1237                          struct ring_buffer_event *rbe, void *key,
1238                          struct action_data *data, u64 *var_ref_vals)
1239 {
1240         struct synth_event *event = data->synth_event;
1241
1242         trace_synth(event, var_ref_vals, data->var_ref_idx);
1243 }
1244
1245 struct hist_var_data {
1246         struct list_head list;
1247         struct hist_trigger_data *hist_data;
1248 };
1249
1250 static int __create_synth_event(int argc, const char *name, const char **argv)
1251 {
1252         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1253         struct synth_event *event = NULL;
1254         int i, consumed = 0, n_fields = 0, ret = 0;
1255
1256         /*
1257          * Argument syntax:
1258          *  - Add synthetic event: <event_name> field[;field] ...
1259          *  - Remove synthetic event: !<event_name> field[;field] ...
1260          *      where 'field' = type field_name
1261          */
1262
1263         if (name[0] == '\0' || argc < 1)
1264                 return -EINVAL;
1265
1266         mutex_lock(&event_mutex);
1267
1268         event = find_synth_event(name);
1269         if (event) {
1270                 ret = -EEXIST;
1271                 goto out;
1272         }
1273
1274         for (i = 0; i < argc - 1; i++) {
1275                 if (strcmp(argv[i], ";") == 0)
1276                         continue;
1277                 if (n_fields == SYNTH_FIELDS_MAX) {
1278                         ret = -EINVAL;
1279                         goto err;
1280                 }
1281
1282                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1283                 if (IS_ERR(field)) {
1284                         ret = PTR_ERR(field);
1285                         goto err;
1286                 }
1287                 fields[n_fields++] = field;
1288                 i += consumed - 1;
1289         }
1290
1291         if (i < argc && strcmp(argv[i], ";") != 0) {
1292                 ret = -EINVAL;
1293                 goto err;
1294         }
1295
1296         event = alloc_synth_event(name, n_fields, fields);
1297         if (IS_ERR(event)) {
1298                 ret = PTR_ERR(event);
1299                 event = NULL;
1300                 goto err;
1301         }
1302         ret = register_synth_event(event);
1303         if (!ret)
1304                 dyn_event_add(&event->devent);
1305         else
1306                 free_synth_event(event);
1307  out:
1308         mutex_unlock(&event_mutex);
1309
1310         return ret;
1311  err:
1312         for (i = 0; i < n_fields; i++)
1313                 free_synth_field(fields[i]);
1314
1315         goto out;
1316 }
1317
1318 static int create_or_delete_synth_event(int argc, char **argv)
1319 {
1320         const char *name = argv[0];
1321         struct synth_event *event = NULL;
1322         int ret;
1323
1324         /* trace_run_command() ensures argc != 0 */
1325         if (name[0] == '!') {
1326                 mutex_lock(&event_mutex);
1327                 event = find_synth_event(name + 1);
1328                 if (event) {
1329                         if (event->ref)
1330                                 ret = -EBUSY;
1331                         else {
1332                                 ret = unregister_synth_event(event);
1333                                 if (!ret) {
1334                                         dyn_event_remove(&event->devent);
1335                                         free_synth_event(event);
1336                                 }
1337                         }
1338                 } else
1339                         ret = -ENOENT;
1340                 mutex_unlock(&event_mutex);
1341                 return ret;
1342         }
1343
1344         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1345         return ret == -ECANCELED ? -EINVAL : ret;
1346 }
1347
1348 static int synth_event_create(int argc, const char **argv)
1349 {
1350         const char *name = argv[0];
1351         int len;
1352
1353         if (name[0] != 's' || name[1] != ':')
1354                 return -ECANCELED;
1355         name += 2;
1356
1357         /* This interface accepts group name prefix */
1358         if (strchr(name, '/')) {
1359                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
1360                 if (len == 0)
1361                         return -EINVAL;
1362                 name += len;
1363         }
1364         return __create_synth_event(argc - 1, name, argv + 1);
1365 }
1366
1367 static int synth_event_release(struct dyn_event *ev)
1368 {
1369         struct synth_event *event = to_synth_event(ev);
1370         int ret;
1371
1372         if (event->ref)
1373                 return -EBUSY;
1374
1375         ret = unregister_synth_event(event);
1376         if (ret)
1377                 return ret;
1378
1379         dyn_event_remove(ev);
1380         free_synth_event(event);
1381         return 0;
1382 }
1383
1384 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1385 {
1386         struct synth_field *field;
1387         unsigned int i;
1388
1389         seq_printf(m, "%s\t", event->name);
1390
1391         for (i = 0; i < event->n_fields; i++) {
1392                 field = event->fields[i];
1393
1394                 /* parameter values */
1395                 seq_printf(m, "%s %s%s", field->type, field->name,
1396                            i == event->n_fields - 1 ? "" : "; ");
1397         }
1398
1399         seq_putc(m, '\n');
1400
1401         return 0;
1402 }
1403
1404 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1405 {
1406         struct synth_event *event = to_synth_event(ev);
1407
1408         seq_printf(m, "s:%s/", event->class.system);
1409
1410         return __synth_event_show(m, event);
1411 }
1412
1413 static int synth_events_seq_show(struct seq_file *m, void *v)
1414 {
1415         struct dyn_event *ev = v;
1416
1417         if (!is_synth_event(ev))
1418                 return 0;
1419
1420         return __synth_event_show(m, to_synth_event(ev));
1421 }
1422
1423 static const struct seq_operations synth_events_seq_op = {
1424         .start  = dyn_event_seq_start,
1425         .next   = dyn_event_seq_next,
1426         .stop   = dyn_event_seq_stop,
1427         .show   = synth_events_seq_show,
1428 };
1429
1430 static int synth_events_open(struct inode *inode, struct file *file)
1431 {
1432         int ret;
1433
1434         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1435                 ret = dyn_events_release_all(&synth_event_ops);
1436                 if (ret < 0)
1437                         return ret;
1438         }
1439
1440         return seq_open(file, &synth_events_seq_op);
1441 }
1442
1443 static ssize_t synth_events_write(struct file *file,
1444                                   const char __user *buffer,
1445                                   size_t count, loff_t *ppos)
1446 {
1447         return trace_parse_run_command(file, buffer, count, ppos,
1448                                        create_or_delete_synth_event);
1449 }
1450
1451 static const struct file_operations synth_events_fops = {
1452         .open           = synth_events_open,
1453         .write          = synth_events_write,
1454         .read           = seq_read,
1455         .llseek         = seq_lseek,
1456         .release        = seq_release,
1457 };
1458
1459 static u64 hist_field_timestamp(struct hist_field *hist_field,
1460                                 struct tracing_map_elt *elt,
1461                                 struct ring_buffer_event *rbe,
1462                                 void *event)
1463 {
1464         struct hist_trigger_data *hist_data = hist_field->hist_data;
1465         struct trace_array *tr = hist_data->event_file->tr;
1466
1467         u64 ts = ring_buffer_event_time_stamp(rbe);
1468
1469         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1470                 ts = ns2usecs(ts);
1471
1472         return ts;
1473 }
1474
1475 static u64 hist_field_cpu(struct hist_field *hist_field,
1476                           struct tracing_map_elt *elt,
1477                           struct ring_buffer_event *rbe,
1478                           void *event)
1479 {
1480         int cpu = smp_processor_id();
1481
1482         return cpu;
1483 }
1484
1485 /**
1486  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1487  * @hist_field: The VAR_REF field to check
1488  * @var_data: The hist trigger that owns the variable
1489  * @var_idx: The trigger variable identifier
1490  *
1491  * Check the given VAR_REF field to see whether or not it references
1492  * the given variable associated with the given trigger.
1493  *
1494  * Return: The VAR_REF field if it does reference the variable, NULL if not
1495  */
1496 static struct hist_field *
1497 check_field_for_var_ref(struct hist_field *hist_field,
1498                         struct hist_trigger_data *var_data,
1499                         unsigned int var_idx)
1500 {
1501         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1502
1503         if (hist_field && hist_field->var.idx == var_idx &&
1504             hist_field->var.hist_data == var_data)
1505                 return hist_field;
1506
1507         return NULL;
1508 }
1509
1510 /**
1511  * find_var_ref - Check if a trigger has a reference to a trigger variable
1512  * @hist_data: The hist trigger that might have a reference to the variable
1513  * @var_data: The hist trigger that owns the variable
1514  * @var_idx: The trigger variable identifier
1515  *
1516  * Check the list of var_refs[] on the first hist trigger to see
1517  * whether any of them are references to the variable on the second
1518  * trigger.
1519  *
1520  * Return: The VAR_REF field referencing the variable if so, NULL if not
1521  */
1522 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1523                                        struct hist_trigger_data *var_data,
1524                                        unsigned int var_idx)
1525 {
1526         struct hist_field *hist_field;
1527         unsigned int i;
1528
1529         for (i = 0; i < hist_data->n_var_refs; i++) {
1530                 hist_field = hist_data->var_refs[i];
1531                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1532                         return hist_field;
1533         }
1534
1535         return NULL;
1536 }
1537
1538 /**
1539  * find_any_var_ref - Check if there is a reference to a given trigger variable
1540  * @hist_data: The hist trigger
1541  * @var_idx: The trigger variable identifier
1542  *
1543  * Check to see whether the given variable is currently referenced by
1544  * any other trigger.
1545  *
1546  * The trigger the variable is defined on is explicitly excluded - the
1547  * assumption being that a self-reference doesn't prevent a trigger
1548  * from being removed.
1549  *
1550  * Return: The VAR_REF field referencing the variable if so, NULL if not
1551  */
1552 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1553                                            unsigned int var_idx)
1554 {
1555         struct trace_array *tr = hist_data->event_file->tr;
1556         struct hist_field *found = NULL;
1557         struct hist_var_data *var_data;
1558
1559         list_for_each_entry(var_data, &tr->hist_vars, list) {
1560                 if (var_data->hist_data == hist_data)
1561                         continue;
1562                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1563                 if (found)
1564                         break;
1565         }
1566
1567         return found;
1568 }
1569
1570 /**
1571  * check_var_refs - Check if there is a reference to any of trigger's variables
1572  * @hist_data: The hist trigger
1573  *
1574  * A trigger can define one or more variables.  If any one of them is
1575  * currently referenced by any other trigger, this function will
1576  * determine that.
1577
1578  * Typically used to determine whether or not a trigger can be removed
1579  * - if there are any references to a trigger's variables, it cannot.
1580  *
1581  * Return: True if there is a reference to any of trigger's variables
1582  */
1583 static bool check_var_refs(struct hist_trigger_data *hist_data)
1584 {
1585         struct hist_field *field;
1586         bool found = false;
1587         int i;
1588
1589         for_each_hist_field(i, hist_data) {
1590                 field = hist_data->fields[i];
1591                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1592                         if (find_any_var_ref(hist_data, field->var.idx)) {
1593                                 found = true;
1594                                 break;
1595                         }
1596                 }
1597         }
1598
1599         return found;
1600 }
1601
1602 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1603 {
1604         struct trace_array *tr = hist_data->event_file->tr;
1605         struct hist_var_data *var_data, *found = NULL;
1606
1607         list_for_each_entry(var_data, &tr->hist_vars, list) {
1608                 if (var_data->hist_data == hist_data) {
1609                         found = var_data;
1610                         break;
1611                 }
1612         }
1613
1614         return found;
1615 }
1616
1617 static bool field_has_hist_vars(struct hist_field *hist_field,
1618                                 unsigned int level)
1619 {
1620         int i;
1621
1622         if (level > 3)
1623                 return false;
1624
1625         if (!hist_field)
1626                 return false;
1627
1628         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1629             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1630                 return true;
1631
1632         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1633                 struct hist_field *operand;
1634
1635                 operand = hist_field->operands[i];
1636                 if (field_has_hist_vars(operand, level + 1))
1637                         return true;
1638         }
1639
1640         return false;
1641 }
1642
1643 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1644 {
1645         struct hist_field *hist_field;
1646         int i;
1647
1648         for_each_hist_field(i, hist_data) {
1649                 hist_field = hist_data->fields[i];
1650                 if (field_has_hist_vars(hist_field, 0))
1651                         return true;
1652         }
1653
1654         return false;
1655 }
1656
1657 static int save_hist_vars(struct hist_trigger_data *hist_data)
1658 {
1659         struct trace_array *tr = hist_data->event_file->tr;
1660         struct hist_var_data *var_data;
1661
1662         var_data = find_hist_vars(hist_data);
1663         if (var_data)
1664                 return 0;
1665
1666         if (trace_array_get(tr) < 0)
1667                 return -ENODEV;
1668
1669         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1670         if (!var_data) {
1671                 trace_array_put(tr);
1672                 return -ENOMEM;
1673         }
1674
1675         var_data->hist_data = hist_data;
1676         list_add(&var_data->list, &tr->hist_vars);
1677
1678         return 0;
1679 }
1680
1681 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1682 {
1683         struct trace_array *tr = hist_data->event_file->tr;
1684         struct hist_var_data *var_data;
1685
1686         var_data = find_hist_vars(hist_data);
1687         if (!var_data)
1688                 return;
1689
1690         if (WARN_ON(check_var_refs(hist_data)))
1691                 return;
1692
1693         list_del(&var_data->list);
1694
1695         kfree(var_data);
1696
1697         trace_array_put(tr);
1698 }
1699
1700 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1701                                          const char *var_name)
1702 {
1703         struct hist_field *hist_field, *found = NULL;
1704         int i;
1705
1706         for_each_hist_field(i, hist_data) {
1707                 hist_field = hist_data->fields[i];
1708                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1709                     strcmp(hist_field->var.name, var_name) == 0) {
1710                         found = hist_field;
1711                         break;
1712                 }
1713         }
1714
1715         return found;
1716 }
1717
1718 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1719                                    struct trace_event_file *file,
1720                                    const char *var_name)
1721 {
1722         struct hist_trigger_data *test_data;
1723         struct event_trigger_data *test;
1724         struct hist_field *hist_field;
1725
1726         hist_field = find_var_field(hist_data, var_name);
1727         if (hist_field)
1728                 return hist_field;
1729
1730         list_for_each_entry_rcu(test, &file->triggers, list) {
1731                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1732                         test_data = test->private_data;
1733                         hist_field = find_var_field(test_data, var_name);
1734                         if (hist_field)
1735                                 return hist_field;
1736                 }
1737         }
1738
1739         return NULL;
1740 }
1741
1742 static struct trace_event_file *find_var_file(struct trace_array *tr,
1743                                               char *system,
1744                                               char *event_name,
1745                                               char *var_name)
1746 {
1747         struct hist_trigger_data *var_hist_data;
1748         struct hist_var_data *var_data;
1749         struct trace_event_file *file, *found = NULL;
1750
1751         if (system)
1752                 return find_event_file(tr, system, event_name);
1753
1754         list_for_each_entry(var_data, &tr->hist_vars, list) {
1755                 var_hist_data = var_data->hist_data;
1756                 file = var_hist_data->event_file;
1757                 if (file == found)
1758                         continue;
1759
1760                 if (find_var_field(var_hist_data, var_name)) {
1761                         if (found) {
1762                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1763                                 return NULL;
1764                         }
1765
1766                         found = file;
1767                 }
1768         }
1769
1770         return found;
1771 }
1772
1773 static struct hist_field *find_file_var(struct trace_event_file *file,
1774                                         const char *var_name)
1775 {
1776         struct hist_trigger_data *test_data;
1777         struct event_trigger_data *test;
1778         struct hist_field *hist_field;
1779
1780         list_for_each_entry_rcu(test, &file->triggers, list) {
1781                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1782                         test_data = test->private_data;
1783                         hist_field = find_var_field(test_data, var_name);
1784                         if (hist_field)
1785                                 return hist_field;
1786                 }
1787         }
1788
1789         return NULL;
1790 }
1791
1792 static struct hist_field *
1793 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1794 {
1795         struct trace_array *tr = hist_data->event_file->tr;
1796         struct hist_field *hist_field, *found = NULL;
1797         struct trace_event_file *file;
1798         unsigned int i;
1799
1800         for (i = 0; i < hist_data->n_actions; i++) {
1801                 struct action_data *data = hist_data->actions[i];
1802
1803                 if (data->handler == HANDLER_ONMATCH) {
1804                         char *system = data->match_data.event_system;
1805                         char *event_name = data->match_data.event;
1806
1807                         file = find_var_file(tr, system, event_name, var_name);
1808                         if (!file)
1809                                 continue;
1810                         hist_field = find_file_var(file, var_name);
1811                         if (hist_field) {
1812                                 if (found) {
1813                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1814                                                  errpos(var_name));
1815                                         return ERR_PTR(-EINVAL);
1816                                 }
1817
1818                                 found = hist_field;
1819                         }
1820                 }
1821         }
1822         return found;
1823 }
1824
1825 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1826                                          char *system,
1827                                          char *event_name,
1828                                          char *var_name)
1829 {
1830         struct trace_array *tr = hist_data->event_file->tr;
1831         struct hist_field *hist_field = NULL;
1832         struct trace_event_file *file;
1833
1834         if (!system || !event_name) {
1835                 hist_field = find_match_var(hist_data, var_name);
1836                 if (IS_ERR(hist_field))
1837                         return NULL;
1838                 if (hist_field)
1839                         return hist_field;
1840         }
1841
1842         file = find_var_file(tr, system, event_name, var_name);
1843         if (!file)
1844                 return NULL;
1845
1846         hist_field = find_file_var(file, var_name);
1847
1848         return hist_field;
1849 }
1850
1851 static u64 hist_field_var_ref(struct hist_field *hist_field,
1852                               struct tracing_map_elt *elt,
1853                               struct ring_buffer_event *rbe,
1854                               void *event)
1855 {
1856         struct hist_elt_data *elt_data;
1857         u64 var_val = 0;
1858
1859         if (WARN_ON_ONCE(!elt))
1860                 return var_val;
1861
1862         elt_data = elt->private_data;
1863         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1864
1865         return var_val;
1866 }
1867
1868 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1869                              u64 *var_ref_vals, bool self)
1870 {
1871         struct hist_trigger_data *var_data;
1872         struct tracing_map_elt *var_elt;
1873         struct hist_field *hist_field;
1874         unsigned int i, var_idx;
1875         bool resolved = true;
1876         u64 var_val = 0;
1877
1878         for (i = 0; i < hist_data->n_var_refs; i++) {
1879                 hist_field = hist_data->var_refs[i];
1880                 var_idx = hist_field->var.idx;
1881                 var_data = hist_field->var.hist_data;
1882
1883                 if (var_data == NULL) {
1884                         resolved = false;
1885                         break;
1886                 }
1887
1888                 if ((self && var_data != hist_data) ||
1889                     (!self && var_data == hist_data))
1890                         continue;
1891
1892                 var_elt = tracing_map_lookup(var_data->map, key);
1893                 if (!var_elt) {
1894                         resolved = false;
1895                         break;
1896                 }
1897
1898                 if (!tracing_map_var_set(var_elt, var_idx)) {
1899                         resolved = false;
1900                         break;
1901                 }
1902
1903                 if (self || !hist_field->read_once)
1904                         var_val = tracing_map_read_var(var_elt, var_idx);
1905                 else
1906                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1907
1908                 var_ref_vals[i] = var_val;
1909         }
1910
1911         return resolved;
1912 }
1913
1914 static const char *hist_field_name(struct hist_field *field,
1915                                    unsigned int level)
1916 {
1917         const char *field_name = "";
1918
1919         if (level > 1)
1920                 return field_name;
1921
1922         if (field->field)
1923                 field_name = field->field->name;
1924         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1925                  field->flags & HIST_FIELD_FL_ALIAS)
1926                 field_name = hist_field_name(field->operands[0], ++level);
1927         else if (field->flags & HIST_FIELD_FL_CPU)
1928                 field_name = "cpu";
1929         else if (field->flags & HIST_FIELD_FL_EXPR ||
1930                  field->flags & HIST_FIELD_FL_VAR_REF) {
1931                 if (field->system) {
1932                         static char full_name[MAX_FILTER_STR_VAL];
1933
1934                         strcat(full_name, field->system);
1935                         strcat(full_name, ".");
1936                         strcat(full_name, field->event_name);
1937                         strcat(full_name, ".");
1938                         strcat(full_name, field->name);
1939                         field_name = full_name;
1940                 } else
1941                         field_name = field->name;
1942         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1943                 field_name = "common_timestamp";
1944
1945         if (field_name == NULL)
1946                 field_name = "";
1947
1948         return field_name;
1949 }
1950
1951 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1952 {
1953         hist_field_fn_t fn = NULL;
1954
1955         switch (field_size) {
1956         case 8:
1957                 if (field_is_signed)
1958                         fn = hist_field_s64;
1959                 else
1960                         fn = hist_field_u64;
1961                 break;
1962         case 4:
1963                 if (field_is_signed)
1964                         fn = hist_field_s32;
1965                 else
1966                         fn = hist_field_u32;
1967                 break;
1968         case 2:
1969                 if (field_is_signed)
1970                         fn = hist_field_s16;
1971                 else
1972                         fn = hist_field_u16;
1973                 break;
1974         case 1:
1975                 if (field_is_signed)
1976                         fn = hist_field_s8;
1977                 else
1978                         fn = hist_field_u8;
1979                 break;
1980         }
1981
1982         return fn;
1983 }
1984
1985 static int parse_map_size(char *str)
1986 {
1987         unsigned long size, map_bits;
1988         int ret;
1989
1990         strsep(&str, "=");
1991         if (!str) {
1992                 ret = -EINVAL;
1993                 goto out;
1994         }
1995
1996         ret = kstrtoul(str, 0, &size);
1997         if (ret)
1998                 goto out;
1999
2000         map_bits = ilog2(roundup_pow_of_two(size));
2001         if (map_bits < TRACING_MAP_BITS_MIN ||
2002             map_bits > TRACING_MAP_BITS_MAX)
2003                 ret = -EINVAL;
2004         else
2005                 ret = map_bits;
2006  out:
2007         return ret;
2008 }
2009
2010 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2011 {
2012         unsigned int i;
2013
2014         if (!attrs)
2015                 return;
2016
2017         for (i = 0; i < attrs->n_assignments; i++)
2018                 kfree(attrs->assignment_str[i]);
2019
2020         for (i = 0; i < attrs->n_actions; i++)
2021                 kfree(attrs->action_str[i]);
2022
2023         kfree(attrs->name);
2024         kfree(attrs->sort_key_str);
2025         kfree(attrs->keys_str);
2026         kfree(attrs->vals_str);
2027         kfree(attrs->clock);
2028         kfree(attrs);
2029 }
2030
2031 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2032 {
2033         int ret = -EINVAL;
2034
2035         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2036                 return ret;
2037
2038         if ((str_has_prefix(str, "onmatch(")) ||
2039             (str_has_prefix(str, "onmax(")) ||
2040             (str_has_prefix(str, "onchange("))) {
2041                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2042                 if (!attrs->action_str[attrs->n_actions]) {
2043                         ret = -ENOMEM;
2044                         return ret;
2045                 }
2046                 attrs->n_actions++;
2047                 ret = 0;
2048         }
2049         return ret;
2050 }
2051
2052 static int parse_assignment(struct trace_array *tr,
2053                             char *str, struct hist_trigger_attrs *attrs)
2054 {
2055         int ret = 0;
2056
2057         if ((str_has_prefix(str, "key=")) ||
2058             (str_has_prefix(str, "keys="))) {
2059                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
2060                 if (!attrs->keys_str) {
2061                         ret = -ENOMEM;
2062                         goto out;
2063                 }
2064         } else if ((str_has_prefix(str, "val=")) ||
2065                    (str_has_prefix(str, "vals=")) ||
2066                    (str_has_prefix(str, "values="))) {
2067                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
2068                 if (!attrs->vals_str) {
2069                         ret = -ENOMEM;
2070                         goto out;
2071                 }
2072         } else if (str_has_prefix(str, "sort=")) {
2073                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2074                 if (!attrs->sort_key_str) {
2075                         ret = -ENOMEM;
2076                         goto out;
2077                 }
2078         } else if (str_has_prefix(str, "name=")) {
2079                 attrs->name = kstrdup(str, GFP_KERNEL);
2080                 if (!attrs->name) {
2081                         ret = -ENOMEM;
2082                         goto out;
2083                 }
2084         } else if (str_has_prefix(str, "clock=")) {
2085                 strsep(&str, "=");
2086                 if (!str) {
2087                         ret = -EINVAL;
2088                         goto out;
2089                 }
2090
2091                 str = strstrip(str);
2092                 attrs->clock = kstrdup(str, GFP_KERNEL);
2093                 if (!attrs->clock) {
2094                         ret = -ENOMEM;
2095                         goto out;
2096                 }
2097         } else if (str_has_prefix(str, "size=")) {
2098                 int map_bits = parse_map_size(str);
2099
2100                 if (map_bits < 0) {
2101                         ret = map_bits;
2102                         goto out;
2103                 }
2104                 attrs->map_bits = map_bits;
2105         } else {
2106                 char *assignment;
2107
2108                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2109                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
2110                         ret = -EINVAL;
2111                         goto out;
2112                 }
2113
2114                 assignment = kstrdup(str, GFP_KERNEL);
2115                 if (!assignment) {
2116                         ret = -ENOMEM;
2117                         goto out;
2118                 }
2119
2120                 attrs->assignment_str[attrs->n_assignments++] = assignment;
2121         }
2122  out:
2123         return ret;
2124 }
2125
2126 static struct hist_trigger_attrs *
2127 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
2128 {
2129         struct hist_trigger_attrs *attrs;
2130         int ret = 0;
2131
2132         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2133         if (!attrs)
2134                 return ERR_PTR(-ENOMEM);
2135
2136         while (trigger_str) {
2137                 char *str = strsep(&trigger_str, ":");
2138
2139                 if (strchr(str, '=')) {
2140                         ret = parse_assignment(tr, str, attrs);
2141                         if (ret)
2142                                 goto free;
2143                 } else if (strcmp(str, "pause") == 0)
2144                         attrs->pause = true;
2145                 else if ((strcmp(str, "cont") == 0) ||
2146                          (strcmp(str, "continue") == 0))
2147                         attrs->cont = true;
2148                 else if (strcmp(str, "clear") == 0)
2149                         attrs->clear = true;
2150                 else {
2151                         ret = parse_action(str, attrs);
2152                         if (ret)
2153                                 goto free;
2154                 }
2155         }
2156
2157         if (!attrs->keys_str) {
2158                 ret = -EINVAL;
2159                 goto free;
2160         }
2161
2162         if (!attrs->clock) {
2163                 attrs->clock = kstrdup("global", GFP_KERNEL);
2164                 if (!attrs->clock) {
2165                         ret = -ENOMEM;
2166                         goto free;
2167                 }
2168         }
2169
2170         return attrs;
2171  free:
2172         destroy_hist_trigger_attrs(attrs);
2173
2174         return ERR_PTR(ret);
2175 }
2176
2177 static inline void save_comm(char *comm, struct task_struct *task)
2178 {
2179         if (!task->pid) {
2180                 strcpy(comm, "<idle>");
2181                 return;
2182         }
2183
2184         if (WARN_ON_ONCE(task->pid < 0)) {
2185                 strcpy(comm, "<XXX>");
2186                 return;
2187         }
2188
2189         strncpy(comm, task->comm, TASK_COMM_LEN);
2190 }
2191
2192 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2193 {
2194         unsigned int i;
2195
2196         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2197                 kfree(elt_data->field_var_str[i]);
2198
2199         kfree(elt_data->comm);
2200         kfree(elt_data);
2201 }
2202
2203 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2204 {
2205         struct hist_elt_data *elt_data = elt->private_data;
2206
2207         hist_elt_data_free(elt_data);
2208 }
2209
2210 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2211 {
2212         struct hist_trigger_data *hist_data = elt->map->private_data;
2213         unsigned int size = TASK_COMM_LEN;
2214         struct hist_elt_data *elt_data;
2215         struct hist_field *key_field;
2216         unsigned int i, n_str;
2217
2218         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2219         if (!elt_data)
2220                 return -ENOMEM;
2221
2222         for_each_hist_key_field(i, hist_data) {
2223                 key_field = hist_data->fields[i];
2224
2225                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2226                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2227                         if (!elt_data->comm) {
2228                                 kfree(elt_data);
2229                                 return -ENOMEM;
2230                         }
2231                         break;
2232                 }
2233         }
2234
2235         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2236
2237         size = STR_VAR_LEN_MAX;
2238
2239         for (i = 0; i < n_str; i++) {
2240                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2241                 if (!elt_data->field_var_str[i]) {
2242                         hist_elt_data_free(elt_data);
2243                         return -ENOMEM;
2244                 }
2245         }
2246
2247         elt->private_data = elt_data;
2248
2249         return 0;
2250 }
2251
2252 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2253 {
2254         struct hist_elt_data *elt_data = elt->private_data;
2255
2256         if (elt_data->comm)
2257                 save_comm(elt_data->comm, current);
2258 }
2259
2260 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2261         .elt_alloc      = hist_trigger_elt_data_alloc,
2262         .elt_free       = hist_trigger_elt_data_free,
2263         .elt_init       = hist_trigger_elt_data_init,
2264 };
2265
2266 static const char *get_hist_field_flags(struct hist_field *hist_field)
2267 {
2268         const char *flags_str = NULL;
2269
2270         if (hist_field->flags & HIST_FIELD_FL_HEX)
2271                 flags_str = "hex";
2272         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2273                 flags_str = "sym";
2274         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2275                 flags_str = "sym-offset";
2276         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2277                 flags_str = "execname";
2278         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2279                 flags_str = "syscall";
2280         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2281                 flags_str = "log2";
2282         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2283                 flags_str = "usecs";
2284
2285         return flags_str;
2286 }
2287
2288 static void expr_field_str(struct hist_field *field, char *expr)
2289 {
2290         if (field->flags & HIST_FIELD_FL_VAR_REF)
2291                 strcat(expr, "$");
2292
2293         strcat(expr, hist_field_name(field, 0));
2294
2295         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2296                 const char *flags_str = get_hist_field_flags(field);
2297
2298                 if (flags_str) {
2299                         strcat(expr, ".");
2300                         strcat(expr, flags_str);
2301                 }
2302         }
2303 }
2304
2305 static char *expr_str(struct hist_field *field, unsigned int level)
2306 {
2307         char *expr;
2308
2309         if (level > 1)
2310                 return NULL;
2311
2312         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2313         if (!expr)
2314                 return NULL;
2315
2316         if (!field->operands[0]) {
2317                 expr_field_str(field, expr);
2318                 return expr;
2319         }
2320
2321         if (field->operator == FIELD_OP_UNARY_MINUS) {
2322                 char *subexpr;
2323
2324                 strcat(expr, "-(");
2325                 subexpr = expr_str(field->operands[0], ++level);
2326                 if (!subexpr) {
2327                         kfree(expr);
2328                         return NULL;
2329                 }
2330                 strcat(expr, subexpr);
2331                 strcat(expr, ")");
2332
2333                 kfree(subexpr);
2334
2335                 return expr;
2336         }
2337
2338         expr_field_str(field->operands[0], expr);
2339
2340         switch (field->operator) {
2341         case FIELD_OP_MINUS:
2342                 strcat(expr, "-");
2343                 break;
2344         case FIELD_OP_PLUS:
2345                 strcat(expr, "+");
2346                 break;
2347         default:
2348                 kfree(expr);
2349                 return NULL;
2350         }
2351
2352         expr_field_str(field->operands[1], expr);
2353
2354         return expr;
2355 }
2356
2357 static int contains_operator(char *str)
2358 {
2359         enum field_op_id field_op = FIELD_OP_NONE;
2360         char *op;
2361
2362         op = strpbrk(str, "+-");
2363         if (!op)
2364                 return FIELD_OP_NONE;
2365
2366         switch (*op) {
2367         case '-':
2368                 if (*str == '-')
2369                         field_op = FIELD_OP_UNARY_MINUS;
2370                 else
2371                         field_op = FIELD_OP_MINUS;
2372                 break;
2373         case '+':
2374                 field_op = FIELD_OP_PLUS;
2375                 break;
2376         default:
2377                 break;
2378         }
2379
2380         return field_op;
2381 }
2382
2383 static void __destroy_hist_field(struct hist_field *hist_field)
2384 {
2385         kfree(hist_field->var.name);
2386         kfree(hist_field->name);
2387         kfree(hist_field->type);
2388
2389         kfree(hist_field);
2390 }
2391
2392 static void destroy_hist_field(struct hist_field *hist_field,
2393                                unsigned int level)
2394 {
2395         unsigned int i;
2396
2397         if (level > 3)
2398                 return;
2399
2400         if (!hist_field)
2401                 return;
2402
2403         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2404                 return; /* var refs will be destroyed separately */
2405
2406         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2407                 destroy_hist_field(hist_field->operands[i], level + 1);
2408
2409         __destroy_hist_field(hist_field);
2410 }
2411
2412 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2413                                             struct ftrace_event_field *field,
2414                                             unsigned long flags,
2415                                             char *var_name)
2416 {
2417         struct hist_field *hist_field;
2418
2419         if (field && is_function_field(field))
2420                 return NULL;
2421
2422         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2423         if (!hist_field)
2424                 return NULL;
2425
2426         hist_field->hist_data = hist_data;
2427
2428         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2429                 goto out; /* caller will populate */
2430
2431         if (flags & HIST_FIELD_FL_VAR_REF) {
2432                 hist_field->fn = hist_field_var_ref;
2433                 goto out;
2434         }
2435
2436         if (flags & HIST_FIELD_FL_HITCOUNT) {
2437                 hist_field->fn = hist_field_counter;
2438                 hist_field->size = sizeof(u64);
2439                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2440                 if (!hist_field->type)
2441                         goto free;
2442                 goto out;
2443         }
2444
2445         if (flags & HIST_FIELD_FL_STACKTRACE) {
2446                 hist_field->fn = hist_field_none;
2447                 goto out;
2448         }
2449
2450         if (flags & HIST_FIELD_FL_LOG2) {
2451                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2452                 hist_field->fn = hist_field_log2;
2453                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2454                 hist_field->size = hist_field->operands[0]->size;
2455                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2456                 if (!hist_field->type)
2457                         goto free;
2458                 goto out;
2459         }
2460
2461         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2462                 hist_field->fn = hist_field_timestamp;
2463                 hist_field->size = sizeof(u64);
2464                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2465                 if (!hist_field->type)
2466                         goto free;
2467                 goto out;
2468         }
2469
2470         if (flags & HIST_FIELD_FL_CPU) {
2471                 hist_field->fn = hist_field_cpu;
2472                 hist_field->size = sizeof(int);
2473                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2474                 if (!hist_field->type)
2475                         goto free;
2476                 goto out;
2477         }
2478
2479         if (WARN_ON_ONCE(!field))
2480                 goto out;
2481
2482         if (is_string_field(field)) {
2483                 flags |= HIST_FIELD_FL_STRING;
2484
2485                 hist_field->size = MAX_FILTER_STR_VAL;
2486                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2487                 if (!hist_field->type)
2488                         goto free;
2489
2490                 if (field->filter_type == FILTER_STATIC_STRING)
2491                         hist_field->fn = hist_field_string;
2492                 else if (field->filter_type == FILTER_DYN_STRING)
2493                         hist_field->fn = hist_field_dynstring;
2494                 else
2495                         hist_field->fn = hist_field_pstring;
2496         } else {
2497                 hist_field->size = field->size;
2498                 hist_field->is_signed = field->is_signed;
2499                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2500                 if (!hist_field->type)
2501                         goto free;
2502
2503                 hist_field->fn = select_value_fn(field->size,
2504                                                  field->is_signed);
2505                 if (!hist_field->fn) {
2506                         destroy_hist_field(hist_field, 0);
2507                         return NULL;
2508                 }
2509         }
2510  out:
2511         hist_field->field = field;
2512         hist_field->flags = flags;
2513
2514         if (var_name) {
2515                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2516                 if (!hist_field->var.name)
2517                         goto free;
2518         }
2519
2520         return hist_field;
2521  free:
2522         destroy_hist_field(hist_field, 0);
2523         return NULL;
2524 }
2525
2526 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2527 {
2528         unsigned int i;
2529
2530         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2531                 if (hist_data->fields[i]) {
2532                         destroy_hist_field(hist_data->fields[i], 0);
2533                         hist_data->fields[i] = NULL;
2534                 }
2535         }
2536
2537         for (i = 0; i < hist_data->n_var_refs; i++) {
2538                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2539                 __destroy_hist_field(hist_data->var_refs[i]);
2540                 hist_data->var_refs[i] = NULL;
2541         }
2542 }
2543
2544 static int init_var_ref(struct hist_field *ref_field,
2545                         struct hist_field *var_field,
2546                         char *system, char *event_name)
2547 {
2548         int err = 0;
2549
2550         ref_field->var.idx = var_field->var.idx;
2551         ref_field->var.hist_data = var_field->hist_data;
2552         ref_field->size = var_field->size;
2553         ref_field->is_signed = var_field->is_signed;
2554         ref_field->flags |= var_field->flags &
2555                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2556
2557         if (system) {
2558                 ref_field->system = kstrdup(system, GFP_KERNEL);
2559                 if (!ref_field->system)
2560                         return -ENOMEM;
2561         }
2562
2563         if (event_name) {
2564                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2565                 if (!ref_field->event_name) {
2566                         err = -ENOMEM;
2567                         goto free;
2568                 }
2569         }
2570
2571         if (var_field->var.name) {
2572                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2573                 if (!ref_field->name) {
2574                         err = -ENOMEM;
2575                         goto free;
2576                 }
2577         } else if (var_field->name) {
2578                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2579                 if (!ref_field->name) {
2580                         err = -ENOMEM;
2581                         goto free;
2582                 }
2583         }
2584
2585         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2586         if (!ref_field->type) {
2587                 err = -ENOMEM;
2588                 goto free;
2589         }
2590  out:
2591         return err;
2592  free:
2593         kfree(ref_field->system);
2594         kfree(ref_field->event_name);
2595         kfree(ref_field->name);
2596
2597         goto out;
2598 }
2599
2600 /**
2601  * create_var_ref - Create a variable reference and attach it to trigger
2602  * @hist_data: The trigger that will be referencing the variable
2603  * @var_field: The VAR field to create a reference to
2604  * @system: The optional system string
2605  * @event_name: The optional event_name string
2606  *
2607  * Given a variable hist_field, create a VAR_REF hist_field that
2608  * represents a reference to it.
2609  *
2610  * This function also adds the reference to the trigger that
2611  * now references the variable.
2612  *
2613  * Return: The VAR_REF field if successful, NULL if not
2614  */
2615 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2616                                          struct hist_field *var_field,
2617                                          char *system, char *event_name)
2618 {
2619         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2620         struct hist_field *ref_field;
2621
2622         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2623         if (ref_field) {
2624                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2625                         destroy_hist_field(ref_field, 0);
2626                         return NULL;
2627                 }
2628
2629                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2630                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2631         }
2632
2633         return ref_field;
2634 }
2635
2636 static bool is_var_ref(char *var_name)
2637 {
2638         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2639                 return false;
2640
2641         return true;
2642 }
2643
2644 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2645                                  char *var_name)
2646 {
2647         char *name, *field;
2648         unsigned int i;
2649
2650         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2651                 name = hist_data->attrs->var_defs.name[i];
2652
2653                 if (strcmp(var_name, name) == 0) {
2654                         field = hist_data->attrs->var_defs.expr[i];
2655                         if (contains_operator(field) || is_var_ref(field))
2656                                 continue;
2657                         return field;
2658                 }
2659         }
2660
2661         return NULL;
2662 }
2663
2664 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2665                                  char *system, char *event_name,
2666                                  char *var_name)
2667 {
2668         struct trace_event_call *call;
2669
2670         if (system && event_name) {
2671                 call = hist_data->event_file->event_call;
2672
2673                 if (strcmp(system, call->class->system) != 0)
2674                         return NULL;
2675
2676                 if (strcmp(event_name, trace_event_name(call)) != 0)
2677                         return NULL;
2678         }
2679
2680         if (!!system != !!event_name)
2681                 return NULL;
2682
2683         if (!is_var_ref(var_name))
2684                 return NULL;
2685
2686         var_name++;
2687
2688         return field_name_from_var(hist_data, var_name);
2689 }
2690
2691 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2692                                         char *system, char *event_name,
2693                                         char *var_name)
2694 {
2695         struct hist_field *var_field = NULL, *ref_field = NULL;
2696         struct trace_array *tr = hist_data->event_file->tr;
2697
2698         if (!is_var_ref(var_name))
2699                 return NULL;
2700
2701         var_name++;
2702
2703         var_field = find_event_var(hist_data, system, event_name, var_name);
2704         if (var_field)
2705                 ref_field = create_var_ref(hist_data, var_field,
2706                                            system, event_name);
2707
2708         if (!ref_field)
2709                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2710
2711         return ref_field;
2712 }
2713
2714 static struct ftrace_event_field *
2715 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2716             char *field_str, unsigned long *flags)
2717 {
2718         struct ftrace_event_field *field = NULL;
2719         char *field_name, *modifier, *str;
2720         struct trace_array *tr = file->tr;
2721
2722         modifier = str = kstrdup(field_str, GFP_KERNEL);
2723         if (!modifier)
2724                 return ERR_PTR(-ENOMEM);
2725
2726         field_name = strsep(&modifier, ".");
2727         if (modifier) {
2728                 if (strcmp(modifier, "hex") == 0)
2729                         *flags |= HIST_FIELD_FL_HEX;
2730                 else if (strcmp(modifier, "sym") == 0)
2731                         *flags |= HIST_FIELD_FL_SYM;
2732                 else if (strcmp(modifier, "sym-offset") == 0)
2733                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2734                 else if ((strcmp(modifier, "execname") == 0) &&
2735                          (strcmp(field_name, "common_pid") == 0))
2736                         *flags |= HIST_FIELD_FL_EXECNAME;
2737                 else if (strcmp(modifier, "syscall") == 0)
2738                         *flags |= HIST_FIELD_FL_SYSCALL;
2739                 else if (strcmp(modifier, "log2") == 0)
2740                         *flags |= HIST_FIELD_FL_LOG2;
2741                 else if (strcmp(modifier, "usecs") == 0)
2742                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2743                 else {
2744                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2745                         field = ERR_PTR(-EINVAL);
2746                         goto out;
2747                 }
2748         }
2749
2750         if (strcmp(field_name, "common_timestamp") == 0) {
2751                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2752                 hist_data->enable_timestamps = true;
2753                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2754                         hist_data->attrs->ts_in_usecs = true;
2755         } else if (strcmp(field_name, "cpu") == 0)
2756                 *flags |= HIST_FIELD_FL_CPU;
2757         else {
2758                 field = trace_find_event_field(file->event_call, field_name);
2759                 if (!field || !field->size) {
2760                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
2761                         field = ERR_PTR(-EINVAL);
2762                         goto out;
2763                 }
2764         }
2765  out:
2766         kfree(str);
2767
2768         return field;
2769 }
2770
2771 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2772                                        struct hist_field *var_ref,
2773                                        char *var_name)
2774 {
2775         struct hist_field *alias = NULL;
2776         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2777
2778         alias = create_hist_field(hist_data, NULL, flags, var_name);
2779         if (!alias)
2780                 return NULL;
2781
2782         alias->fn = var_ref->fn;
2783         alias->operands[0] = var_ref;
2784
2785         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2786                 destroy_hist_field(alias, 0);
2787                 return NULL;
2788         }
2789
2790         alias->var_ref_idx = var_ref->var_ref_idx;
2791
2792         return alias;
2793 }
2794
2795 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2796                                      struct trace_event_file *file, char *str,
2797                                      unsigned long *flags, char *var_name)
2798 {
2799         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2800         struct ftrace_event_field *field = NULL;
2801         struct hist_field *hist_field = NULL;
2802         int ret = 0;
2803
2804         s = strchr(str, '.');
2805         if (s) {
2806                 s = strchr(++s, '.');
2807                 if (s) {
2808                         ref_system = strsep(&str, ".");
2809                         if (!str) {
2810                                 ret = -EINVAL;
2811                                 goto out;
2812                         }
2813                         ref_event = strsep(&str, ".");
2814                         if (!str) {
2815                                 ret = -EINVAL;
2816                                 goto out;
2817                         }
2818                         ref_var = str;
2819                 }
2820         }
2821
2822         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2823         if (!s) {
2824                 hist_field = parse_var_ref(hist_data, ref_system,
2825                                            ref_event, ref_var);
2826                 if (hist_field) {
2827                         if (var_name) {
2828                                 hist_field = create_alias(hist_data, hist_field, var_name);
2829                                 if (!hist_field) {
2830                                         ret = -ENOMEM;
2831                                         goto out;
2832                                 }
2833                         }
2834                         return hist_field;
2835                 }
2836         } else
2837                 str = s;
2838
2839         field = parse_field(hist_data, file, str, flags);
2840         if (IS_ERR(field)) {
2841                 ret = PTR_ERR(field);
2842                 goto out;
2843         }
2844
2845         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2846         if (!hist_field) {
2847                 ret = -ENOMEM;
2848                 goto out;
2849         }
2850
2851         return hist_field;
2852  out:
2853         return ERR_PTR(ret);
2854 }
2855
2856 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2857                                      struct trace_event_file *file,
2858                                      char *str, unsigned long flags,
2859                                      char *var_name, unsigned int level);
2860
2861 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2862                                       struct trace_event_file *file,
2863                                       char *str, unsigned long flags,
2864                                       char *var_name, unsigned int level)
2865 {
2866         struct hist_field *operand1, *expr = NULL;
2867         unsigned long operand_flags;
2868         int ret = 0;
2869         char *s;
2870
2871         /* we support only -(xxx) i.e. explicit parens required */
2872
2873         if (level > 3) {
2874                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2875                 ret = -EINVAL;
2876                 goto free;
2877         }
2878
2879         str++; /* skip leading '-' */
2880
2881         s = strchr(str, '(');
2882         if (s)
2883                 str++;
2884         else {
2885                 ret = -EINVAL;
2886                 goto free;
2887         }
2888
2889         s = strrchr(str, ')');
2890         if (s)
2891                 *s = '\0';
2892         else {
2893                 ret = -EINVAL; /* no closing ')' */
2894                 goto free;
2895         }
2896
2897         flags |= HIST_FIELD_FL_EXPR;
2898         expr = create_hist_field(hist_data, NULL, flags, var_name);
2899         if (!expr) {
2900                 ret = -ENOMEM;
2901                 goto free;
2902         }
2903
2904         operand_flags = 0;
2905         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2906         if (IS_ERR(operand1)) {
2907                 ret = PTR_ERR(operand1);
2908                 goto free;
2909         }
2910
2911         expr->flags |= operand1->flags &
2912                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2913         expr->fn = hist_field_unary_minus;
2914         expr->operands[0] = operand1;
2915         expr->operator = FIELD_OP_UNARY_MINUS;
2916         expr->name = expr_str(expr, 0);
2917         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2918         if (!expr->type) {
2919                 ret = -ENOMEM;
2920                 goto free;
2921         }
2922
2923         return expr;
2924  free:
2925         destroy_hist_field(expr, 0);
2926         return ERR_PTR(ret);
2927 }
2928
2929 static int check_expr_operands(struct trace_array *tr,
2930                                struct hist_field *operand1,
2931                                struct hist_field *operand2)
2932 {
2933         unsigned long operand1_flags = operand1->flags;
2934         unsigned long operand2_flags = operand2->flags;
2935
2936         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2937             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2938                 struct hist_field *var;
2939
2940                 var = find_var_field(operand1->var.hist_data, operand1->name);
2941                 if (!var)
2942                         return -EINVAL;
2943                 operand1_flags = var->flags;
2944         }
2945
2946         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2947             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2948                 struct hist_field *var;
2949
2950                 var = find_var_field(operand2->var.hist_data, operand2->name);
2951                 if (!var)
2952                         return -EINVAL;
2953                 operand2_flags = var->flags;
2954         }
2955
2956         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2957             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2958                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2959                 return -EINVAL;
2960         }
2961
2962         return 0;
2963 }
2964
2965 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2966                                      struct trace_event_file *file,
2967                                      char *str, unsigned long flags,
2968                                      char *var_name, unsigned int level)
2969 {
2970         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2971         unsigned long operand_flags;
2972         int field_op, ret = -EINVAL;
2973         char *sep, *operand1_str;
2974
2975         if (level > 3) {
2976                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2977                 return ERR_PTR(-EINVAL);
2978         }
2979
2980         field_op = contains_operator(str);
2981
2982         if (field_op == FIELD_OP_NONE)
2983                 return parse_atom(hist_data, file, str, &flags, var_name);
2984
2985         if (field_op == FIELD_OP_UNARY_MINUS)
2986                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2987
2988         switch (field_op) {
2989         case FIELD_OP_MINUS:
2990                 sep = "-";
2991                 break;
2992         case FIELD_OP_PLUS:
2993                 sep = "+";
2994                 break;
2995         default:
2996                 goto free;
2997         }
2998
2999         operand1_str = strsep(&str, sep);
3000         if (!operand1_str || !str)
3001                 goto free;
3002
3003         operand_flags = 0;
3004         operand1 = parse_atom(hist_data, file, operand1_str,
3005                               &operand_flags, NULL);
3006         if (IS_ERR(operand1)) {
3007                 ret = PTR_ERR(operand1);
3008                 operand1 = NULL;
3009                 goto free;
3010         }
3011
3012         /* rest of string could be another expression e.g. b+c in a+b+c */
3013         operand_flags = 0;
3014         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3015         if (IS_ERR(operand2)) {
3016                 ret = PTR_ERR(operand2);
3017                 operand2 = NULL;
3018                 goto free;
3019         }
3020
3021         ret = check_expr_operands(file->tr, operand1, operand2);
3022         if (ret)
3023                 goto free;
3024
3025         flags |= HIST_FIELD_FL_EXPR;
3026
3027         flags |= operand1->flags &
3028                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3029
3030         expr = create_hist_field(hist_data, NULL, flags, var_name);
3031         if (!expr) {
3032                 ret = -ENOMEM;
3033                 goto free;
3034         }
3035
3036         operand1->read_once = true;
3037         operand2->read_once = true;
3038
3039         expr->operands[0] = operand1;
3040         expr->operands[1] = operand2;
3041         expr->operator = field_op;
3042         expr->name = expr_str(expr, 0);
3043         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3044         if (!expr->type) {
3045                 ret = -ENOMEM;
3046                 goto free;
3047         }
3048
3049         switch (field_op) {
3050         case FIELD_OP_MINUS:
3051                 expr->fn = hist_field_minus;
3052                 break;
3053         case FIELD_OP_PLUS:
3054                 expr->fn = hist_field_plus;
3055                 break;
3056         default:
3057                 ret = -EINVAL;
3058                 goto free;
3059         }
3060
3061         return expr;
3062  free:
3063         destroy_hist_field(operand1, 0);
3064         destroy_hist_field(operand2, 0);
3065         destroy_hist_field(expr, 0);
3066
3067         return ERR_PTR(ret);
3068 }
3069
3070 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3071                                  struct trace_event_file *file)
3072 {
3073         struct event_trigger_data *test;
3074
3075         list_for_each_entry_rcu(test, &file->triggers, list) {
3076                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3077                         if (test->private_data == hist_data)
3078                                 return test->filter_str;
3079                 }
3080         }
3081
3082         return NULL;
3083 }
3084
3085 static struct event_command trigger_hist_cmd;
3086 static int event_hist_trigger_func(struct event_command *cmd_ops,
3087                                    struct trace_event_file *file,
3088                                    char *glob, char *cmd, char *param);
3089
3090 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3091                             struct hist_trigger_data *hist_data,
3092                             unsigned int n_keys)
3093 {
3094         struct hist_field *target_hist_field, *hist_field;
3095         unsigned int n, i, j;
3096
3097         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3098                 return false;
3099
3100         i = hist_data->n_vals;
3101         j = target_hist_data->n_vals;
3102
3103         for (n = 0; n < n_keys; n++) {
3104                 hist_field = hist_data->fields[i + n];
3105                 target_hist_field = target_hist_data->fields[j + n];
3106
3107                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3108                         return false;
3109                 if (hist_field->size != target_hist_field->size)
3110                         return false;
3111                 if (hist_field->is_signed != target_hist_field->is_signed)
3112                         return false;
3113         }
3114
3115         return true;
3116 }
3117
3118 static struct hist_trigger_data *
3119 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3120                      struct trace_event_file *file)
3121 {
3122         struct hist_trigger_data *hist_data;
3123         struct event_trigger_data *test;
3124         unsigned int n_keys;
3125
3126         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3127
3128         list_for_each_entry_rcu(test, &file->triggers, list) {
3129                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3130                         hist_data = test->private_data;
3131
3132                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3133                                 return hist_data;
3134                 }
3135         }
3136
3137         return NULL;
3138 }
3139
3140 static struct trace_event_file *event_file(struct trace_array *tr,
3141                                            char *system, char *event_name)
3142 {
3143         struct trace_event_file *file;
3144
3145         file = __find_event_file(tr, system, event_name);
3146         if (!file)
3147                 return ERR_PTR(-EINVAL);
3148
3149         return file;
3150 }
3151
3152 static struct hist_field *
3153 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3154                          char *system, char *event_name, char *field_name)
3155 {
3156         struct hist_field *event_var;
3157         char *synthetic_name;
3158
3159         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3160         if (!synthetic_name)
3161                 return ERR_PTR(-ENOMEM);
3162
3163         strcpy(synthetic_name, "synthetic_");
3164         strcat(synthetic_name, field_name);
3165
3166         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3167
3168         kfree(synthetic_name);
3169
3170         return event_var;
3171 }
3172
3173 /**
3174  * create_field_var_hist - Automatically create a histogram and var for a field
3175  * @target_hist_data: The target hist trigger
3176  * @subsys_name: Optional subsystem name
3177  * @event_name: Optional event name
3178  * @field_name: The name of the field (and the resulting variable)
3179  *
3180  * Hist trigger actions fetch data from variables, not directly from
3181  * events.  However, for convenience, users are allowed to directly
3182  * specify an event field in an action, which will be automatically
3183  * converted into a variable on their behalf.
3184
3185  * If a user specifies a field on an event that isn't the event the
3186  * histogram currently being defined (the target event histogram), the
3187  * only way that can be accomplished is if a new hist trigger is
3188  * created and the field variable defined on that.
3189  *
3190  * This function creates a new histogram compatible with the target
3191  * event (meaning a histogram with the same key as the target
3192  * histogram), and creates a variable for the specified field, but
3193  * with 'synthetic_' prepended to the variable name in order to avoid
3194  * collision with normal field variables.
3195  *
3196  * Return: The variable created for the field.
3197  */
3198 static struct hist_field *
3199 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3200                       char *subsys_name, char *event_name, char *field_name)
3201 {
3202         struct trace_array *tr = target_hist_data->event_file->tr;
3203         struct hist_field *event_var = ERR_PTR(-EINVAL);
3204         struct hist_trigger_data *hist_data;
3205         unsigned int i, n, first = true;
3206         struct field_var_hist *var_hist;
3207         struct trace_event_file *file;
3208         struct hist_field *key_field;
3209         char *saved_filter;
3210         char *cmd;
3211         int ret;
3212
3213         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3214                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3215                 return ERR_PTR(-EINVAL);
3216         }
3217
3218         file = event_file(tr, subsys_name, event_name);
3219
3220         if (IS_ERR(file)) {
3221                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3222                 ret = PTR_ERR(file);
3223                 return ERR_PTR(ret);
3224         }
3225
3226         /*
3227          * Look for a histogram compatible with target.  We'll use the
3228          * found histogram specification to create a new matching
3229          * histogram with our variable on it.  target_hist_data is not
3230          * yet a registered histogram so we can't use that.
3231          */
3232         hist_data = find_compatible_hist(target_hist_data, file);
3233         if (!hist_data) {
3234                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3235                 return ERR_PTR(-EINVAL);
3236         }
3237
3238         /* See if a synthetic field variable has already been created */
3239         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3240                                              event_name, field_name);
3241         if (!IS_ERR_OR_NULL(event_var))
3242                 return event_var;
3243
3244         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3245         if (!var_hist)
3246                 return ERR_PTR(-ENOMEM);
3247
3248         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3249         if (!cmd) {
3250                 kfree(var_hist);
3251                 return ERR_PTR(-ENOMEM);
3252         }
3253
3254         /* Use the same keys as the compatible histogram */
3255         strcat(cmd, "keys=");
3256
3257         for_each_hist_key_field(i, hist_data) {
3258                 key_field = hist_data->fields[i];
3259                 if (!first)
3260                         strcat(cmd, ",");
3261                 strcat(cmd, key_field->field->name);
3262                 first = false;
3263         }
3264
3265         /* Create the synthetic field variable specification */
3266         strcat(cmd, ":synthetic_");
3267         strcat(cmd, field_name);
3268         strcat(cmd, "=");
3269         strcat(cmd, field_name);
3270
3271         /* Use the same filter as the compatible histogram */
3272         saved_filter = find_trigger_filter(hist_data, file);
3273         if (saved_filter) {
3274                 strcat(cmd, " if ");
3275                 strcat(cmd, saved_filter);
3276         }
3277
3278         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3279         if (!var_hist->cmd) {
3280                 kfree(cmd);
3281                 kfree(var_hist);
3282                 return ERR_PTR(-ENOMEM);
3283         }
3284
3285         /* Save the compatible histogram information */
3286         var_hist->hist_data = hist_data;
3287
3288         /* Create the new histogram with our variable */
3289         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3290                                       "", "hist", cmd);
3291         if (ret) {
3292                 kfree(cmd);
3293                 kfree(var_hist->cmd);
3294                 kfree(var_hist);
3295                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3296                 return ERR_PTR(ret);
3297         }
3298
3299         kfree(cmd);
3300
3301         /* If we can't find the variable, something went wrong */
3302         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3303                                              event_name, field_name);
3304         if (IS_ERR_OR_NULL(event_var)) {
3305                 kfree(var_hist->cmd);
3306                 kfree(var_hist);
3307                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3308                 return ERR_PTR(-EINVAL);
3309         }
3310
3311         n = target_hist_data->n_field_var_hists;
3312         target_hist_data->field_var_hists[n] = var_hist;
3313         target_hist_data->n_field_var_hists++;
3314
3315         return event_var;
3316 }
3317
3318 static struct hist_field *
3319 find_target_event_var(struct hist_trigger_data *hist_data,
3320                       char *subsys_name, char *event_name, char *var_name)
3321 {
3322         struct trace_event_file *file = hist_data->event_file;
3323         struct hist_field *hist_field = NULL;
3324
3325         if (subsys_name) {
3326                 struct trace_event_call *call;
3327
3328                 if (!event_name)
3329                         return NULL;
3330
3331                 call = file->event_call;
3332
3333                 if (strcmp(subsys_name, call->class->system) != 0)
3334                         return NULL;
3335
3336                 if (strcmp(event_name, trace_event_name(call)) != 0)
3337                         return NULL;
3338         }
3339
3340         hist_field = find_var_field(hist_data, var_name);
3341
3342         return hist_field;
3343 }
3344
3345 static inline void __update_field_vars(struct tracing_map_elt *elt,
3346                                        struct ring_buffer_event *rbe,
3347                                        void *rec,
3348                                        struct field_var **field_vars,
3349                                        unsigned int n_field_vars,
3350                                        unsigned int field_var_str_start)
3351 {
3352         struct hist_elt_data *elt_data = elt->private_data;
3353         unsigned int i, j, var_idx;
3354         u64 var_val;
3355
3356         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3357                 struct field_var *field_var = field_vars[i];
3358                 struct hist_field *var = field_var->var;
3359                 struct hist_field *val = field_var->val;
3360
3361                 var_val = val->fn(val, elt, rbe, rec);
3362                 var_idx = var->var.idx;
3363
3364                 if (val->flags & HIST_FIELD_FL_STRING) {
3365                         char *str = elt_data->field_var_str[j++];
3366                         char *val_str = (char *)(uintptr_t)var_val;
3367
3368                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3369                         var_val = (u64)(uintptr_t)str;
3370                 }
3371                 tracing_map_set_var(elt, var_idx, var_val);
3372         }
3373 }
3374
3375 static void update_field_vars(struct hist_trigger_data *hist_data,
3376                               struct tracing_map_elt *elt,
3377                               struct ring_buffer_event *rbe,
3378                               void *rec)
3379 {
3380         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3381                             hist_data->n_field_vars, 0);
3382 }
3383
3384 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3385                                  struct tracing_map_elt *elt, void *rec,
3386                                  struct ring_buffer_event *rbe, void *key,
3387                                  struct action_data *data, u64 *var_ref_vals)
3388 {
3389         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
3390                             hist_data->n_save_vars, hist_data->n_field_var_str);
3391 }
3392
3393 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3394                                      struct trace_event_file *file,
3395                                      char *name, int size, const char *type)
3396 {
3397         struct hist_field *var;
3398         int idx;
3399
3400         if (find_var(hist_data, file, name) && !hist_data->remove) {
3401                 var = ERR_PTR(-EINVAL);
3402                 goto out;
3403         }
3404
3405         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3406         if (!var) {
3407                 var = ERR_PTR(-ENOMEM);
3408                 goto out;
3409         }
3410
3411         idx = tracing_map_add_var(hist_data->map);
3412         if (idx < 0) {
3413                 kfree(var);
3414                 var = ERR_PTR(-EINVAL);
3415                 goto out;
3416         }
3417
3418         var->flags = HIST_FIELD_FL_VAR;
3419         var->var.idx = idx;
3420         var->var.hist_data = var->hist_data = hist_data;
3421         var->size = size;
3422         var->var.name = kstrdup(name, GFP_KERNEL);
3423         var->type = kstrdup(type, GFP_KERNEL);
3424         if (!var->var.name || !var->type) {
3425                 kfree(var->var.name);
3426                 kfree(var->type);
3427                 kfree(var);
3428                 var = ERR_PTR(-ENOMEM);
3429         }
3430  out:
3431         return var;
3432 }
3433
3434 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3435                                           struct trace_event_file *file,
3436                                           char *field_name)
3437 {
3438         struct hist_field *val = NULL, *var = NULL;
3439         unsigned long flags = HIST_FIELD_FL_VAR;
3440         struct trace_array *tr = file->tr;
3441         struct field_var *field_var;
3442         int ret = 0;
3443
3444         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3445                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3446                 ret = -EINVAL;
3447                 goto err;
3448         }
3449
3450         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3451         if (IS_ERR(val)) {
3452                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3453                 ret = PTR_ERR(val);
3454                 goto err;
3455         }
3456
3457         var = create_var(hist_data, file, field_name, val->size, val->type);
3458         if (IS_ERR(var)) {
3459                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3460                 kfree(val);
3461                 ret = PTR_ERR(var);
3462                 goto err;
3463         }
3464
3465         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3466         if (!field_var) {
3467                 kfree(val);
3468                 kfree(var);
3469                 ret =  -ENOMEM;
3470                 goto err;
3471         }
3472
3473         field_var->var = var;
3474         field_var->val = val;
3475  out:
3476         return field_var;
3477  err:
3478         field_var = ERR_PTR(ret);
3479         goto out;
3480 }
3481
3482 /**
3483  * create_target_field_var - Automatically create a variable for a field
3484  * @target_hist_data: The target hist trigger
3485  * @subsys_name: Optional subsystem name
3486  * @event_name: Optional event name
3487  * @var_name: The name of the field (and the resulting variable)
3488  *
3489  * Hist trigger actions fetch data from variables, not directly from
3490  * events.  However, for convenience, users are allowed to directly
3491  * specify an event field in an action, which will be automatically
3492  * converted into a variable on their behalf.
3493
3494  * This function creates a field variable with the name var_name on
3495  * the hist trigger currently being defined on the target event.  If
3496  * subsys_name and event_name are specified, this function simply
3497  * verifies that they do in fact match the target event subsystem and
3498  * event name.
3499  *
3500  * Return: The variable created for the field.
3501  */
3502 static struct field_var *
3503 create_target_field_var(struct hist_trigger_data *target_hist_data,
3504                         char *subsys_name, char *event_name, char *var_name)
3505 {
3506         struct trace_event_file *file = target_hist_data->event_file;
3507
3508         if (subsys_name) {
3509                 struct trace_event_call *call;
3510
3511                 if (!event_name)
3512                         return NULL;
3513
3514                 call = file->event_call;
3515
3516                 if (strcmp(subsys_name, call->class->system) != 0)
3517                         return NULL;
3518
3519                 if (strcmp(event_name, trace_event_name(call)) != 0)
3520                         return NULL;
3521         }
3522
3523         return create_field_var(target_hist_data, file, var_name);
3524 }
3525
3526 static bool check_track_val_max(u64 track_val, u64 var_val)
3527 {
3528         if (var_val <= track_val)
3529                 return false;
3530
3531         return true;
3532 }
3533
3534 static bool check_track_val_changed(u64 track_val, u64 var_val)
3535 {
3536         if (var_val == track_val)
3537                 return false;
3538
3539         return true;
3540 }
3541
3542 static u64 get_track_val(struct hist_trigger_data *hist_data,
3543                          struct tracing_map_elt *elt,
3544                          struct action_data *data)
3545 {
3546         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3547         u64 track_val;
3548
3549         track_val = tracing_map_read_var(elt, track_var_idx);
3550
3551         return track_val;
3552 }
3553
3554 static void save_track_val(struct hist_trigger_data *hist_data,
3555                            struct tracing_map_elt *elt,
3556                            struct action_data *data, u64 var_val)
3557 {
3558         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3559
3560         tracing_map_set_var(elt, track_var_idx, var_val);
3561 }
3562
3563 static void save_track_data(struct hist_trigger_data *hist_data,
3564                             struct tracing_map_elt *elt, void *rec,
3565                             struct ring_buffer_event *rbe, void *key,
3566                             struct action_data *data, u64 *var_ref_vals)
3567 {
3568         if (data->track_data.save_data)
3569                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3570 }
3571
3572 static bool check_track_val(struct tracing_map_elt *elt,
3573                             struct action_data *data,
3574                             u64 var_val)
3575 {
3576         struct hist_trigger_data *hist_data;
3577         u64 track_val;
3578
3579         hist_data = data->track_data.track_var->hist_data;
3580         track_val = get_track_val(hist_data, elt, data);
3581
3582         return data->track_data.check_val(track_val, var_val);
3583 }
3584
3585 #ifdef CONFIG_TRACER_SNAPSHOT
3586 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3587 {
3588         /* called with tr->max_lock held */
3589         struct track_data *track_data = tr->cond_snapshot->cond_data;
3590         struct hist_elt_data *elt_data, *track_elt_data;
3591         struct snapshot_context *context = cond_data;
3592         struct action_data *action;
3593         u64 track_val;
3594
3595         if (!track_data)
3596                 return false;
3597
3598         action = track_data->action_data;
3599
3600         track_val = get_track_val(track_data->hist_data, context->elt,
3601                                   track_data->action_data);
3602
3603         if (!action->track_data.check_val(track_data->track_val, track_val))
3604                 return false;
3605
3606         track_data->track_val = track_val;
3607         memcpy(track_data->key, context->key, track_data->key_len);
3608
3609         elt_data = context->elt->private_data;
3610         track_elt_data = track_data->elt.private_data;
3611         if (elt_data->comm)
3612                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3613
3614         track_data->updated = true;
3615
3616         return true;
3617 }
3618
3619 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3620                                      struct tracing_map_elt *elt, void *rec,
3621                                      struct ring_buffer_event *rbe, void *key,
3622                                      struct action_data *data,
3623                                      u64 *var_ref_vals)
3624 {
3625         struct trace_event_file *file = hist_data->event_file;
3626         struct snapshot_context context;
3627
3628         context.elt = elt;
3629         context.key = key;
3630
3631         tracing_snapshot_cond(file->tr, &context);
3632 }
3633
3634 static void hist_trigger_print_key(struct seq_file *m,
3635                                    struct hist_trigger_data *hist_data,
3636                                    void *key,
3637                                    struct tracing_map_elt *elt);
3638
3639 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3640 {
3641         unsigned int i;
3642
3643         if (!hist_data->n_actions)
3644                 return NULL;
3645
3646         for (i = 0; i < hist_data->n_actions; i++) {
3647                 struct action_data *data = hist_data->actions[i];
3648
3649                 if (data->action == ACTION_SNAPSHOT)
3650                         return data;
3651         }
3652
3653         return NULL;
3654 }
3655
3656 static void track_data_snapshot_print(struct seq_file *m,
3657                                       struct hist_trigger_data *hist_data)
3658 {
3659         struct trace_event_file *file = hist_data->event_file;
3660         struct track_data *track_data;
3661         struct action_data *action;
3662
3663         track_data = tracing_cond_snapshot_data(file->tr);
3664         if (!track_data)
3665                 return;
3666
3667         if (!track_data->updated)
3668                 return;
3669
3670         action = snapshot_action(hist_data);
3671         if (!action)
3672                 return;
3673
3674         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3675         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3676                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3677                    action->track_data.var_str, track_data->track_val);
3678
3679         seq_puts(m, "\ttriggered by event with key: ");
3680         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3681         seq_putc(m, '\n');
3682 }
3683 #else
3684 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3685 {
3686         return false;
3687 }
3688 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3689                                      struct tracing_map_elt *elt, void *rec,
3690                                      struct ring_buffer_event *rbe, void *key,
3691                                      struct action_data *data,
3692                                      u64 *var_ref_vals) {}
3693 static void track_data_snapshot_print(struct seq_file *m,
3694                                       struct hist_trigger_data *hist_data) {}
3695 #endif /* CONFIG_TRACER_SNAPSHOT */
3696
3697 static void track_data_print(struct seq_file *m,
3698                              struct hist_trigger_data *hist_data,
3699                              struct tracing_map_elt *elt,
3700                              struct action_data *data)
3701 {
3702         u64 track_val = get_track_val(hist_data, elt, data);
3703         unsigned int i, save_var_idx;
3704
3705         if (data->handler == HANDLER_ONMAX)
3706                 seq_printf(m, "\n\tmax: %10llu", track_val);
3707         else if (data->handler == HANDLER_ONCHANGE)
3708                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3709
3710         if (data->action == ACTION_SNAPSHOT)
3711                 return;
3712
3713         for (i = 0; i < hist_data->n_save_vars; i++) {
3714                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3715                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3716                 u64 val;
3717
3718                 save_var_idx = save_var->var.idx;
3719
3720                 val = tracing_map_read_var(elt, save_var_idx);
3721
3722                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3723                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3724                                    (char *)(uintptr_t)(val));
3725                 } else
3726                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3727         }
3728 }
3729
3730 static void ontrack_action(struct hist_trigger_data *hist_data,
3731                            struct tracing_map_elt *elt, void *rec,
3732                            struct ring_buffer_event *rbe, void *key,
3733                            struct action_data *data, u64 *var_ref_vals)
3734 {
3735         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3736
3737         if (check_track_val(elt, data, var_val)) {
3738                 save_track_val(hist_data, elt, data, var_val);
3739                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3740         }
3741 }
3742
3743 static void action_data_destroy(struct action_data *data)
3744 {
3745         unsigned int i;
3746
3747         lockdep_assert_held(&event_mutex);
3748
3749         kfree(data->action_name);
3750
3751         for (i = 0; i < data->n_params; i++)
3752                 kfree(data->params[i]);
3753
3754         if (data->synth_event)
3755                 data->synth_event->ref--;
3756
3757         kfree(data->synth_event_name);
3758
3759         kfree(data);
3760 }
3761
3762 static void track_data_destroy(struct hist_trigger_data *hist_data,
3763                                struct action_data *data)
3764 {
3765         struct trace_event_file *file = hist_data->event_file;
3766
3767         destroy_hist_field(data->track_data.track_var, 0);
3768
3769         if (data->action == ACTION_SNAPSHOT) {
3770                 struct track_data *track_data;
3771
3772                 track_data = tracing_cond_snapshot_data(file->tr);
3773                 if (track_data && track_data->hist_data == hist_data) {
3774                         tracing_snapshot_cond_disable(file->tr);
3775                         track_data_free(track_data);
3776                 }
3777         }
3778
3779         kfree(data->track_data.var_str);
3780
3781         action_data_destroy(data);
3782 }
3783
3784 static int action_create(struct hist_trigger_data *hist_data,
3785                          struct action_data *data);
3786
3787 static int track_data_create(struct hist_trigger_data *hist_data,
3788                              struct action_data *data)
3789 {
3790         struct hist_field *var_field, *ref_field, *track_var = NULL;
3791         struct trace_event_file *file = hist_data->event_file;
3792         struct trace_array *tr = file->tr;
3793         char *track_data_var_str;
3794         int ret = 0;
3795
3796         track_data_var_str = data->track_data.var_str;
3797         if (track_data_var_str[0] != '$') {
3798                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3799                 return -EINVAL;
3800         }
3801         track_data_var_str++;
3802
3803         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3804         if (!var_field) {
3805                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3806                 return -EINVAL;
3807         }
3808
3809         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3810         if (!ref_field)
3811                 return -ENOMEM;
3812
3813         data->track_data.var_ref = ref_field;
3814
3815         if (data->handler == HANDLER_ONMAX)
3816                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3817         if (IS_ERR(track_var)) {
3818                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3819                 ret = PTR_ERR(track_var);
3820                 goto out;
3821         }
3822
3823         if (data->handler == HANDLER_ONCHANGE)
3824                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3825         if (IS_ERR(track_var)) {
3826                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3827                 ret = PTR_ERR(track_var);
3828                 goto out;
3829         }
3830         data->track_data.track_var = track_var;
3831
3832         ret = action_create(hist_data, data);
3833  out:
3834         return ret;
3835 }
3836
3837 static int parse_action_params(struct trace_array *tr, char *params,
3838                                struct action_data *data)
3839 {
3840         char *param, *saved_param;
3841         bool first_param = true;
3842         int ret = 0;
3843
3844         while (params) {
3845                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3846                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3847                         goto out;
3848                 }
3849
3850                 param = strsep(&params, ",");
3851                 if (!param) {
3852                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3853                         ret = -EINVAL;
3854                         goto out;
3855                 }
3856
3857                 param = strstrip(param);
3858                 if (strlen(param) < 2) {
3859                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3860                         ret = -EINVAL;
3861                         goto out;
3862                 }
3863
3864                 saved_param = kstrdup(param, GFP_KERNEL);
3865                 if (!saved_param) {
3866                         ret = -ENOMEM;
3867                         goto out;
3868                 }
3869
3870                 if (first_param && data->use_trace_keyword) {
3871                         data->synth_event_name = saved_param;
3872                         first_param = false;
3873                         continue;
3874                 }
3875                 first_param = false;
3876
3877                 data->params[data->n_params++] = saved_param;
3878         }
3879  out:
3880         return ret;
3881 }
3882
3883 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3884                         enum handler_id handler)
3885 {
3886         char *action_name;
3887         int ret = 0;
3888
3889         strsep(&str, ".");
3890         if (!str) {
3891                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3892                 ret = -EINVAL;
3893                 goto out;
3894         }
3895
3896         action_name = strsep(&str, "(");
3897         if (!action_name || !str) {
3898                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3899                 ret = -EINVAL;
3900                 goto out;
3901         }
3902
3903         if (str_has_prefix(action_name, "save")) {
3904                 char *params = strsep(&str, ")");
3905
3906                 if (!params) {
3907                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3908                         ret = -EINVAL;
3909                         goto out;
3910                 }
3911
3912                 ret = parse_action_params(tr, params, data);
3913                 if (ret)
3914                         goto out;
3915
3916                 if (handler == HANDLER_ONMAX)
3917                         data->track_data.check_val = check_track_val_max;
3918                 else if (handler == HANDLER_ONCHANGE)
3919                         data->track_data.check_val = check_track_val_changed;
3920                 else {
3921                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3922                         ret = -EINVAL;
3923                         goto out;
3924                 }
3925
3926                 data->track_data.save_data = save_track_data_vars;
3927                 data->fn = ontrack_action;
3928                 data->action = ACTION_SAVE;
3929         } else if (str_has_prefix(action_name, "snapshot")) {
3930                 char *params = strsep(&str, ")");
3931
3932                 if (!str) {
3933                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3934                         ret = -EINVAL;
3935                         goto out;
3936                 }
3937
3938                 if (handler == HANDLER_ONMAX)
3939                         data->track_data.check_val = check_track_val_max;
3940                 else if (handler == HANDLER_ONCHANGE)
3941                         data->track_data.check_val = check_track_val_changed;
3942                 else {
3943                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3944                         ret = -EINVAL;
3945                         goto out;
3946                 }
3947
3948                 data->track_data.save_data = save_track_data_snapshot;
3949                 data->fn = ontrack_action;
3950                 data->action = ACTION_SNAPSHOT;
3951         } else {
3952                 char *params = strsep(&str, ")");
3953
3954                 if (str_has_prefix(action_name, "trace"))
3955                         data->use_trace_keyword = true;
3956
3957                 if (params) {
3958                         ret = parse_action_params(tr, params, data);
3959                         if (ret)
3960                                 goto out;
3961                 }
3962
3963                 if (handler == HANDLER_ONMAX)
3964                         data->track_data.check_val = check_track_val_max;
3965                 else if (handler == HANDLER_ONCHANGE)
3966                         data->track_data.check_val = check_track_val_changed;
3967
3968                 if (handler != HANDLER_ONMATCH) {
3969                         data->track_data.save_data = action_trace;
3970                         data->fn = ontrack_action;
3971                 } else
3972                         data->fn = action_trace;
3973
3974                 data->action = ACTION_TRACE;
3975         }
3976
3977         data->action_name = kstrdup(action_name, GFP_KERNEL);
3978         if (!data->action_name) {
3979                 ret = -ENOMEM;
3980                 goto out;
3981         }
3982
3983         data->handler = handler;
3984  out:
3985         return ret;
3986 }
3987
3988 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3989                                             char *str, enum handler_id handler)
3990 {
3991         struct action_data *data;
3992         int ret = -EINVAL;
3993         char *var_str;
3994
3995         data = kzalloc(sizeof(*data), GFP_KERNEL);
3996         if (!data)
3997                 return ERR_PTR(-ENOMEM);
3998
3999         var_str = strsep(&str, ")");
4000         if (!var_str || !str) {
4001                 ret = -EINVAL;
4002                 goto free;
4003         }
4004
4005         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4006         if (!data->track_data.var_str) {
4007                 ret = -ENOMEM;
4008                 goto free;
4009         }
4010
4011         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4012         if (ret)
4013                 goto free;
4014  out:
4015         return data;
4016  free:
4017         track_data_destroy(hist_data, data);
4018         data = ERR_PTR(ret);
4019         goto out;
4020 }
4021
4022 static void onmatch_destroy(struct action_data *data)
4023 {
4024         kfree(data->match_data.event);
4025         kfree(data->match_data.event_system);
4026
4027         action_data_destroy(data);
4028 }
4029
4030 static void destroy_field_var(struct field_var *field_var)
4031 {
4032         if (!field_var)
4033                 return;
4034
4035         destroy_hist_field(field_var->var, 0);
4036         destroy_hist_field(field_var->val, 0);
4037
4038         kfree(field_var);
4039 }
4040
4041 static void destroy_field_vars(struct hist_trigger_data *hist_data)
4042 {
4043         unsigned int i;
4044
4045         for (i = 0; i < hist_data->n_field_vars; i++)
4046                 destroy_field_var(hist_data->field_vars[i]);
4047 }
4048
4049 static void save_field_var(struct hist_trigger_data *hist_data,
4050                            struct field_var *field_var)
4051 {
4052         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
4053
4054         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4055                 hist_data->n_field_var_str++;
4056 }
4057
4058
4059 static int check_synth_field(struct synth_event *event,
4060                              struct hist_field *hist_field,
4061                              unsigned int field_pos)
4062 {
4063         struct synth_field *field;
4064
4065         if (field_pos >= event->n_fields)
4066                 return -EINVAL;
4067
4068         field = event->fields[field_pos];
4069
4070         if (strcmp(field->type, hist_field->type) != 0)
4071                 return -EINVAL;
4072
4073         return 0;
4074 }
4075
4076 static struct hist_field *
4077 trace_action_find_var(struct hist_trigger_data *hist_data,
4078                       struct action_data *data,
4079                       char *system, char *event, char *var)
4080 {
4081         struct trace_array *tr = hist_data->event_file->tr;
4082         struct hist_field *hist_field;
4083
4084         var++; /* skip '$' */
4085
4086         hist_field = find_target_event_var(hist_data, system, event, var);
4087         if (!hist_field) {
4088                 if (!system && data->handler == HANDLER_ONMATCH) {
4089                         system = data->match_data.event_system;
4090                         event = data->match_data.event;
4091                 }
4092
4093                 hist_field = find_event_var(hist_data, system, event, var);
4094         }
4095
4096         if (!hist_field)
4097                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
4098
4099         return hist_field;
4100 }
4101
4102 static struct hist_field *
4103 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4104                               struct action_data *data, char *system,
4105                               char *event, char *var)
4106 {
4107         struct hist_field *hist_field = NULL;
4108         struct field_var *field_var;
4109
4110         /*
4111          * First try to create a field var on the target event (the
4112          * currently being defined).  This will create a variable for
4113          * unqualified fields on the target event, or if qualified,
4114          * target fields that have qualified names matching the target.
4115          */
4116         field_var = create_target_field_var(hist_data, system, event, var);
4117
4118         if (field_var && !IS_ERR(field_var)) {
4119                 save_field_var(hist_data, field_var);
4120                 hist_field = field_var->var;
4121         } else {
4122                 field_var = NULL;
4123                 /*
4124                  * If no explicit system.event is specfied, default to
4125                  * looking for fields on the onmatch(system.event.xxx)
4126                  * event.
4127                  */
4128                 if (!system && data->handler == HANDLER_ONMATCH) {
4129                         system = data->match_data.event_system;
4130                         event = data->match_data.event;
4131                 }
4132
4133                 /*
4134                  * At this point, we're looking at a field on another
4135                  * event.  Because we can't modify a hist trigger on
4136                  * another event to add a variable for a field, we need
4137                  * to create a new trigger on that event and create the
4138                  * variable at the same time.
4139                  */
4140                 hist_field = create_field_var_hist(hist_data, system, event, var);
4141                 if (IS_ERR(hist_field))
4142                         goto free;
4143         }
4144  out:
4145         return hist_field;
4146  free:
4147         destroy_field_var(field_var);
4148         hist_field = NULL;
4149         goto out;
4150 }
4151
4152 static int trace_action_create(struct hist_trigger_data *hist_data,
4153                                struct action_data *data)
4154 {
4155         struct trace_array *tr = hist_data->event_file->tr;
4156         char *event_name, *param, *system = NULL;
4157         struct hist_field *hist_field, *var_ref;
4158         unsigned int i, var_ref_idx;
4159         unsigned int field_pos = 0;
4160         struct synth_event *event;
4161         char *synth_event_name;
4162         int ret = 0;
4163
4164         lockdep_assert_held(&event_mutex);
4165
4166         if (data->use_trace_keyword)
4167                 synth_event_name = data->synth_event_name;
4168         else
4169                 synth_event_name = data->action_name;
4170
4171         event = find_synth_event(synth_event_name);
4172         if (!event) {
4173                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4174                 return -EINVAL;
4175         }
4176
4177         event->ref++;
4178
4179         var_ref_idx = hist_data->n_var_refs;
4180
4181         for (i = 0; i < data->n_params; i++) {
4182                 char *p;
4183
4184                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4185                 if (!param) {
4186                         ret = -ENOMEM;
4187                         goto err;
4188                 }
4189
4190                 system = strsep(&param, ".");
4191                 if (!param) {
4192                         param = (char *)system;
4193                         system = event_name = NULL;
4194                 } else {
4195                         event_name = strsep(&param, ".");
4196                         if (!param) {
4197                                 kfree(p);
4198                                 ret = -EINVAL;
4199                                 goto err;
4200                         }
4201                 }
4202
4203                 if (param[0] == '$')
4204                         hist_field = trace_action_find_var(hist_data, data,
4205                                                            system, event_name,
4206                                                            param);
4207                 else
4208                         hist_field = trace_action_create_field_var(hist_data,
4209                                                                    data,
4210                                                                    system,
4211                                                                    event_name,
4212                                                                    param);
4213
4214                 if (!hist_field) {
4215                         kfree(p);
4216                         ret = -EINVAL;
4217                         goto err;
4218                 }
4219
4220                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4221                         var_ref = create_var_ref(hist_data, hist_field,
4222                                                  system, event_name);
4223                         if (!var_ref) {
4224                                 kfree(p);
4225                                 ret = -ENOMEM;
4226                                 goto err;
4227                         }
4228
4229                         field_pos++;
4230                         kfree(p);
4231                         continue;
4232                 }
4233
4234                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4235                 kfree(p);
4236                 ret = -EINVAL;
4237                 goto err;
4238         }
4239
4240         if (field_pos != event->n_fields) {
4241                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4242                 ret = -EINVAL;
4243                 goto err;
4244         }
4245
4246         data->synth_event = event;
4247         data->var_ref_idx = var_ref_idx;
4248  out:
4249         return ret;
4250  err:
4251         event->ref--;
4252
4253         goto out;
4254 }
4255
4256 static int action_create(struct hist_trigger_data *hist_data,
4257                          struct action_data *data)
4258 {
4259         struct trace_event_file *file = hist_data->event_file;
4260         struct trace_array *tr = file->tr;
4261         struct track_data *track_data;
4262         struct field_var *field_var;
4263         unsigned int i;
4264         char *param;
4265         int ret = 0;
4266
4267         if (data->action == ACTION_TRACE)
4268                 return trace_action_create(hist_data, data);
4269
4270         if (data->action == ACTION_SNAPSHOT) {
4271                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4272                 if (IS_ERR(track_data)) {
4273                         ret = PTR_ERR(track_data);
4274                         goto out;
4275                 }
4276
4277                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4278                                                    cond_snapshot_update);
4279                 if (ret)
4280                         track_data_free(track_data);
4281
4282                 goto out;
4283         }
4284
4285         if (data->action == ACTION_SAVE) {
4286                 if (hist_data->n_save_vars) {
4287                         ret = -EEXIST;
4288                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4289                         goto out;
4290                 }
4291
4292                 for (i = 0; i < data->n_params; i++) {
4293                         param = kstrdup(data->params[i], GFP_KERNEL);
4294                         if (!param) {
4295                                 ret = -ENOMEM;
4296                                 goto out;
4297                         }
4298
4299                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4300                         if (IS_ERR(field_var)) {
4301                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4302                                          errpos(param));
4303                                 ret = PTR_ERR(field_var);
4304                                 kfree(param);
4305                                 goto out;
4306                         }
4307
4308                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4309                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4310                                 hist_data->n_save_var_str++;
4311                         kfree(param);
4312                 }
4313         }
4314  out:
4315         return ret;
4316 }
4317
4318 static int onmatch_create(struct hist_trigger_data *hist_data,
4319                           struct action_data *data)
4320 {
4321         return action_create(hist_data, data);
4322 }
4323
4324 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4325 {
4326         char *match_event, *match_event_system;
4327         struct action_data *data;
4328         int ret = -EINVAL;
4329
4330         data = kzalloc(sizeof(*data), GFP_KERNEL);
4331         if (!data)
4332                 return ERR_PTR(-ENOMEM);
4333
4334         match_event = strsep(&str, ")");
4335         if (!match_event || !str) {
4336                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4337                 goto free;
4338         }
4339
4340         match_event_system = strsep(&match_event, ".");
4341         if (!match_event) {
4342                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4343                 goto free;
4344         }
4345
4346         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4347                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4348                 goto free;
4349         }
4350
4351         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4352         if (!data->match_data.event) {
4353                 ret = -ENOMEM;
4354                 goto free;
4355         }
4356
4357         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4358         if (!data->match_data.event_system) {
4359                 ret = -ENOMEM;
4360                 goto free;
4361         }
4362
4363         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4364         if (ret)
4365                 goto free;
4366  out:
4367         return data;
4368  free:
4369         onmatch_destroy(data);
4370         data = ERR_PTR(ret);
4371         goto out;
4372 }
4373
4374 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4375 {
4376         hist_data->fields[HITCOUNT_IDX] =
4377                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4378         if (!hist_data->fields[HITCOUNT_IDX])
4379                 return -ENOMEM;
4380
4381         hist_data->n_vals++;
4382         hist_data->n_fields++;
4383
4384         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4385                 return -EINVAL;
4386
4387         return 0;
4388 }
4389
4390 static int __create_val_field(struct hist_trigger_data *hist_data,
4391                               unsigned int val_idx,
4392                               struct trace_event_file *file,
4393                               char *var_name, char *field_str,
4394                               unsigned long flags)
4395 {
4396         struct hist_field *hist_field;
4397         int ret = 0;
4398
4399         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4400         if (IS_ERR(hist_field)) {
4401                 ret = PTR_ERR(hist_field);
4402                 goto out;
4403         }
4404
4405         hist_data->fields[val_idx] = hist_field;
4406
4407         ++hist_data->n_vals;
4408         ++hist_data->n_fields;
4409
4410         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4411                 ret = -EINVAL;
4412  out:
4413         return ret;
4414 }
4415
4416 static int create_val_field(struct hist_trigger_data *hist_data,
4417                             unsigned int val_idx,
4418                             struct trace_event_file *file,
4419                             char *field_str)
4420 {
4421         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4422                 return -EINVAL;
4423
4424         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4425 }
4426
4427 static int create_var_field(struct hist_trigger_data *hist_data,
4428                             unsigned int val_idx,
4429                             struct trace_event_file *file,
4430                             char *var_name, char *expr_str)
4431 {
4432         struct trace_array *tr = hist_data->event_file->tr;
4433         unsigned long flags = 0;
4434
4435         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4436                 return -EINVAL;
4437
4438         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4439                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4440                 return -EINVAL;
4441         }
4442
4443         flags |= HIST_FIELD_FL_VAR;
4444         hist_data->n_vars++;
4445         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4446                 return -EINVAL;
4447
4448         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4449 }
4450
4451 static int create_val_fields(struct hist_trigger_data *hist_data,
4452                              struct trace_event_file *file)
4453 {
4454         char *fields_str, *field_str;
4455         unsigned int i, j = 1;
4456         int ret;
4457
4458         ret = create_hitcount_val(hist_data);
4459         if (ret)
4460                 goto out;
4461
4462         fields_str = hist_data->attrs->vals_str;
4463         if (!fields_str)
4464                 goto out;
4465
4466         strsep(&fields_str, "=");
4467         if (!fields_str)
4468                 goto out;
4469
4470         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4471                      j < TRACING_MAP_VALS_MAX; i++) {
4472                 field_str = strsep(&fields_str, ",");
4473                 if (!field_str)
4474                         break;
4475
4476                 if (strcmp(field_str, "hitcount") == 0)
4477                         continue;
4478
4479                 ret = create_val_field(hist_data, j++, file, field_str);
4480                 if (ret)
4481                         goto out;
4482         }
4483
4484         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4485                 ret = -EINVAL;
4486  out:
4487         return ret;
4488 }
4489
4490 static int create_key_field(struct hist_trigger_data *hist_data,
4491                             unsigned int key_idx,
4492                             unsigned int key_offset,
4493                             struct trace_event_file *file,
4494                             char *field_str)
4495 {
4496         struct trace_array *tr = hist_data->event_file->tr;
4497         struct hist_field *hist_field = NULL;
4498         unsigned long flags = 0;
4499         unsigned int key_size;
4500         int ret = 0;
4501
4502         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4503                 return -EINVAL;
4504
4505         flags |= HIST_FIELD_FL_KEY;
4506
4507         if (strcmp(field_str, "stacktrace") == 0) {
4508                 flags |= HIST_FIELD_FL_STACKTRACE;
4509                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4510                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4511         } else {
4512                 hist_field = parse_expr(hist_data, file, field_str, flags,
4513                                         NULL, 0);
4514                 if (IS_ERR(hist_field)) {
4515                         ret = PTR_ERR(hist_field);
4516                         goto out;
4517                 }
4518
4519                 if (field_has_hist_vars(hist_field, 0)) {
4520                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4521                         destroy_hist_field(hist_field, 0);
4522                         ret = -EINVAL;
4523                         goto out;
4524                 }
4525
4526                 key_size = hist_field->size;
4527         }
4528
4529         hist_data->fields[key_idx] = hist_field;
4530
4531         key_size = ALIGN(key_size, sizeof(u64));
4532         hist_data->fields[key_idx]->size = key_size;
4533         hist_data->fields[key_idx]->offset = key_offset;
4534
4535         hist_data->key_size += key_size;
4536
4537         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4538                 ret = -EINVAL;
4539                 goto out;
4540         }
4541
4542         hist_data->n_keys++;
4543         hist_data->n_fields++;
4544
4545         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4546                 return -EINVAL;
4547
4548         ret = key_size;
4549  out:
4550         return ret;
4551 }
4552
4553 static int create_key_fields(struct hist_trigger_data *hist_data,
4554                              struct trace_event_file *file)
4555 {
4556         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4557         char *fields_str, *field_str;
4558         int ret = -EINVAL;
4559
4560         fields_str = hist_data->attrs->keys_str;
4561         if (!fields_str)
4562                 goto out;
4563
4564         strsep(&fields_str, "=");
4565         if (!fields_str)
4566                 goto out;
4567
4568         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4569                 field_str = strsep(&fields_str, ",");
4570                 if (!field_str)
4571                         break;
4572                 ret = create_key_field(hist_data, i, key_offset,
4573                                        file, field_str);
4574                 if (ret < 0)
4575                         goto out;
4576                 key_offset += ret;
4577         }
4578         if (fields_str) {
4579                 ret = -EINVAL;
4580                 goto out;
4581         }
4582         ret = 0;
4583  out:
4584         return ret;
4585 }
4586
4587 static int create_var_fields(struct hist_trigger_data *hist_data,
4588                              struct trace_event_file *file)
4589 {
4590         unsigned int i, j = hist_data->n_vals;
4591         int ret = 0;
4592
4593         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4594
4595         for (i = 0; i < n_vars; i++) {
4596                 char *var_name = hist_data->attrs->var_defs.name[i];
4597                 char *expr = hist_data->attrs->var_defs.expr[i];
4598
4599                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4600                 if (ret)
4601                         goto out;
4602         }
4603  out:
4604         return ret;
4605 }
4606
4607 static void free_var_defs(struct hist_trigger_data *hist_data)
4608 {
4609         unsigned int i;
4610
4611         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4612                 kfree(hist_data->attrs->var_defs.name[i]);
4613                 kfree(hist_data->attrs->var_defs.expr[i]);
4614         }
4615
4616         hist_data->attrs->var_defs.n_vars = 0;
4617 }
4618
4619 static int parse_var_defs(struct hist_trigger_data *hist_data)
4620 {
4621         struct trace_array *tr = hist_data->event_file->tr;
4622         char *s, *str, *var_name, *field_str;
4623         unsigned int i, j, n_vars = 0;
4624         int ret = 0;
4625
4626         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4627                 str = hist_data->attrs->assignment_str[i];
4628                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4629                         field_str = strsep(&str, ",");
4630                         if (!field_str)
4631                                 break;
4632
4633                         var_name = strsep(&field_str, "=");
4634                         if (!var_name || !field_str) {
4635                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4636                                          errpos(var_name));
4637                                 ret = -EINVAL;
4638                                 goto free;
4639                         }
4640
4641                         if (n_vars == TRACING_MAP_VARS_MAX) {
4642                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4643                                 ret = -EINVAL;
4644                                 goto free;
4645                         }
4646
4647                         s = kstrdup(var_name, GFP_KERNEL);
4648                         if (!s) {
4649                                 ret = -ENOMEM;
4650                                 goto free;
4651                         }
4652                         hist_data->attrs->var_defs.name[n_vars] = s;
4653
4654                         s = kstrdup(field_str, GFP_KERNEL);
4655                         if (!s) {
4656                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4657                                 ret = -ENOMEM;
4658                                 goto free;
4659                         }
4660                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4661
4662                         hist_data->attrs->var_defs.n_vars = n_vars;
4663                 }
4664         }
4665
4666         return ret;
4667  free:
4668         free_var_defs(hist_data);
4669
4670         return ret;
4671 }
4672
4673 static int create_hist_fields(struct hist_trigger_data *hist_data,
4674                               struct trace_event_file *file)
4675 {
4676         int ret;
4677
4678         ret = parse_var_defs(hist_data);
4679         if (ret)
4680                 goto out;
4681
4682         ret = create_val_fields(hist_data, file);
4683         if (ret)
4684                 goto out;
4685
4686         ret = create_var_fields(hist_data, file);
4687         if (ret)
4688                 goto out;
4689
4690         ret = create_key_fields(hist_data, file);
4691         if (ret)
4692                 goto out;
4693  out:
4694         free_var_defs(hist_data);
4695
4696         return ret;
4697 }
4698
4699 static int is_descending(const char *str)
4700 {
4701         if (!str)
4702                 return 0;
4703
4704         if (strcmp(str, "descending") == 0)
4705                 return 1;
4706
4707         if (strcmp(str, "ascending") == 0)
4708                 return 0;
4709
4710         return -EINVAL;
4711 }
4712
4713 static int create_sort_keys(struct hist_trigger_data *hist_data)
4714 {
4715         char *fields_str = hist_data->attrs->sort_key_str;
4716         struct tracing_map_sort_key *sort_key;
4717         int descending, ret = 0;
4718         unsigned int i, j, k;
4719
4720         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4721
4722         if (!fields_str)
4723                 goto out;
4724
4725         strsep(&fields_str, "=");
4726         if (!fields_str) {
4727                 ret = -EINVAL;
4728                 goto out;
4729         }
4730
4731         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4732                 struct hist_field *hist_field;
4733                 char *field_str, *field_name;
4734                 const char *test_name;
4735
4736                 sort_key = &hist_data->sort_keys[i];
4737
4738                 field_str = strsep(&fields_str, ",");
4739                 if (!field_str) {
4740                         if (i == 0)
4741                                 ret = -EINVAL;
4742                         break;
4743                 }
4744
4745                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4746                         ret = -EINVAL;
4747                         break;
4748                 }
4749
4750                 field_name = strsep(&field_str, ".");
4751                 if (!field_name) {
4752                         ret = -EINVAL;
4753                         break;
4754                 }
4755
4756                 if (strcmp(field_name, "hitcount") == 0) {
4757                         descending = is_descending(field_str);
4758                         if (descending < 0) {
4759                                 ret = descending;
4760                                 break;
4761                         }
4762                         sort_key->descending = descending;
4763                         continue;
4764                 }
4765
4766                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4767                         unsigned int idx;
4768
4769                         hist_field = hist_data->fields[j];
4770                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4771                                 continue;
4772
4773                         idx = k++;
4774
4775                         test_name = hist_field_name(hist_field, 0);
4776
4777                         if (strcmp(field_name, test_name) == 0) {
4778                                 sort_key->field_idx = idx;
4779                                 descending = is_descending(field_str);
4780                                 if (descending < 0) {
4781                                         ret = descending;
4782                                         goto out;
4783                                 }
4784                                 sort_key->descending = descending;
4785                                 break;
4786                         }
4787                 }
4788                 if (j == hist_data->n_fields) {
4789                         ret = -EINVAL;
4790                         break;
4791                 }
4792         }
4793
4794         hist_data->n_sort_keys = i;
4795  out:
4796         return ret;
4797 }
4798
4799 static void destroy_actions(struct hist_trigger_data *hist_data)
4800 {
4801         unsigned int i;
4802
4803         for (i = 0; i < hist_data->n_actions; i++) {
4804                 struct action_data *data = hist_data->actions[i];
4805
4806                 if (data->handler == HANDLER_ONMATCH)
4807                         onmatch_destroy(data);
4808                 else if (data->handler == HANDLER_ONMAX ||
4809                          data->handler == HANDLER_ONCHANGE)
4810                         track_data_destroy(hist_data, data);
4811                 else
4812                         kfree(data);
4813         }
4814 }
4815
4816 static int parse_actions(struct hist_trigger_data *hist_data)
4817 {
4818         struct trace_array *tr = hist_data->event_file->tr;
4819         struct action_data *data;
4820         unsigned int i;
4821         int ret = 0;
4822         char *str;
4823         int len;
4824
4825         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4826                 str = hist_data->attrs->action_str[i];
4827
4828                 if ((len = str_has_prefix(str, "onmatch("))) {
4829                         char *action_str = str + len;
4830
4831                         data = onmatch_parse(tr, action_str);
4832                         if (IS_ERR(data)) {
4833                                 ret = PTR_ERR(data);
4834                                 break;
4835                         }
4836                 } else if ((len = str_has_prefix(str, "onmax("))) {
4837                         char *action_str = str + len;
4838
4839                         data = track_data_parse(hist_data, action_str,
4840                                                 HANDLER_ONMAX);
4841                         if (IS_ERR(data)) {
4842                                 ret = PTR_ERR(data);
4843                                 break;
4844                         }
4845                 } else if ((len = str_has_prefix(str, "onchange("))) {
4846                         char *action_str = str + len;
4847
4848                         data = track_data_parse(hist_data, action_str,
4849                                                 HANDLER_ONCHANGE);
4850                         if (IS_ERR(data)) {
4851                                 ret = PTR_ERR(data);
4852                                 break;
4853                         }
4854                 } else {
4855                         ret = -EINVAL;
4856                         break;
4857                 }
4858
4859                 hist_data->actions[hist_data->n_actions++] = data;
4860         }
4861
4862         return ret;
4863 }
4864
4865 static int create_actions(struct hist_trigger_data *hist_data)
4866 {
4867         struct action_data *data;
4868         unsigned int i;
4869         int ret = 0;
4870
4871         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4872                 data = hist_data->actions[i];
4873
4874                 if (data->handler == HANDLER_ONMATCH) {
4875                         ret = onmatch_create(hist_data, data);
4876                         if (ret)
4877                                 break;
4878                 } else if (data->handler == HANDLER_ONMAX ||
4879                            data->handler == HANDLER_ONCHANGE) {
4880                         ret = track_data_create(hist_data, data);
4881                         if (ret)
4882                                 break;
4883                 } else {
4884                         ret = -EINVAL;
4885                         break;
4886                 }
4887         }
4888
4889         return ret;
4890 }
4891
4892 static void print_actions(struct seq_file *m,
4893                           struct hist_trigger_data *hist_data,
4894                           struct tracing_map_elt *elt)
4895 {
4896         unsigned int i;
4897
4898         for (i = 0; i < hist_data->n_actions; i++) {
4899                 struct action_data *data = hist_data->actions[i];
4900
4901                 if (data->action == ACTION_SNAPSHOT)
4902                         continue;
4903
4904                 if (data->handler == HANDLER_ONMAX ||
4905                     data->handler == HANDLER_ONCHANGE)
4906                         track_data_print(m, hist_data, elt, data);
4907         }
4908 }
4909
4910 static void print_action_spec(struct seq_file *m,
4911                               struct hist_trigger_data *hist_data,
4912                               struct action_data *data)
4913 {
4914         unsigned int i;
4915
4916         if (data->action == ACTION_SAVE) {
4917                 for (i = 0; i < hist_data->n_save_vars; i++) {
4918                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4919                         if (i < hist_data->n_save_vars - 1)
4920                                 seq_puts(m, ",");
4921                 }
4922         } else if (data->action == ACTION_TRACE) {
4923                 if (data->use_trace_keyword)
4924                         seq_printf(m, "%s", data->synth_event_name);
4925                 for (i = 0; i < data->n_params; i++) {
4926                         if (i || data->use_trace_keyword)
4927                                 seq_puts(m, ",");
4928                         seq_printf(m, "%s", data->params[i]);
4929                 }
4930         }
4931 }
4932
4933 static void print_track_data_spec(struct seq_file *m,
4934                                   struct hist_trigger_data *hist_data,
4935                                   struct action_data *data)
4936 {
4937         if (data->handler == HANDLER_ONMAX)
4938                 seq_puts(m, ":onmax(");
4939         else if (data->handler == HANDLER_ONCHANGE)
4940                 seq_puts(m, ":onchange(");
4941         seq_printf(m, "%s", data->track_data.var_str);
4942         seq_printf(m, ").%s(", data->action_name);
4943
4944         print_action_spec(m, hist_data, data);
4945
4946         seq_puts(m, ")");
4947 }
4948
4949 static void print_onmatch_spec(struct seq_file *m,
4950                                struct hist_trigger_data *hist_data,
4951                                struct action_data *data)
4952 {
4953         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4954                    data->match_data.event);
4955
4956         seq_printf(m, "%s(", data->action_name);
4957
4958         print_action_spec(m, hist_data, data);
4959
4960         seq_puts(m, ")");
4961 }
4962
4963 static bool actions_match(struct hist_trigger_data *hist_data,
4964                           struct hist_trigger_data *hist_data_test)
4965 {
4966         unsigned int i, j;
4967
4968         if (hist_data->n_actions != hist_data_test->n_actions)
4969                 return false;
4970
4971         for (i = 0; i < hist_data->n_actions; i++) {
4972                 struct action_data *data = hist_data->actions[i];
4973                 struct action_data *data_test = hist_data_test->actions[i];
4974                 char *action_name, *action_name_test;
4975
4976                 if (data->handler != data_test->handler)
4977                         return false;
4978                 if (data->action != data_test->action)
4979                         return false;
4980
4981                 if (data->n_params != data_test->n_params)
4982                         return false;
4983
4984                 for (j = 0; j < data->n_params; j++) {
4985                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4986                                 return false;
4987                 }
4988
4989                 if (data->use_trace_keyword)
4990                         action_name = data->synth_event_name;
4991                 else
4992                         action_name = data->action_name;
4993
4994                 if (data_test->use_trace_keyword)
4995                         action_name_test = data_test->synth_event_name;
4996                 else
4997                         action_name_test = data_test->action_name;
4998
4999                 if (strcmp(action_name, action_name_test) != 0)
5000                         return false;
5001
5002                 if (data->handler == HANDLER_ONMATCH) {
5003                         if (strcmp(data->match_data.event_system,
5004                                    data_test->match_data.event_system) != 0)
5005                                 return false;
5006                         if (strcmp(data->match_data.event,
5007                                    data_test->match_data.event) != 0)
5008                                 return false;
5009                 } else if (data->handler == HANDLER_ONMAX ||
5010                            data->handler == HANDLER_ONCHANGE) {
5011                         if (strcmp(data->track_data.var_str,
5012                                    data_test->track_data.var_str) != 0)
5013                                 return false;
5014                 }
5015         }
5016
5017         return true;
5018 }
5019
5020
5021 static void print_actions_spec(struct seq_file *m,
5022                                struct hist_trigger_data *hist_data)
5023 {
5024         unsigned int i;
5025
5026         for (i = 0; i < hist_data->n_actions; i++) {
5027                 struct action_data *data = hist_data->actions[i];
5028
5029                 if (data->handler == HANDLER_ONMATCH)
5030                         print_onmatch_spec(m, hist_data, data);
5031                 else if (data->handler == HANDLER_ONMAX ||
5032                          data->handler == HANDLER_ONCHANGE)
5033                         print_track_data_spec(m, hist_data, data);
5034         }
5035 }
5036
5037 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5038 {
5039         unsigned int i;
5040
5041         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5042                 kfree(hist_data->field_var_hists[i]->cmd);
5043                 kfree(hist_data->field_var_hists[i]);
5044         }
5045 }
5046
5047 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5048 {
5049         if (!hist_data)
5050                 return;
5051
5052         destroy_hist_trigger_attrs(hist_data->attrs);
5053         destroy_hist_fields(hist_data);
5054         tracing_map_destroy(hist_data->map);
5055
5056         destroy_actions(hist_data);
5057         destroy_field_vars(hist_data);
5058         destroy_field_var_hists(hist_data);
5059
5060         kfree(hist_data);
5061 }
5062
5063 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5064 {
5065         struct tracing_map *map = hist_data->map;
5066         struct ftrace_event_field *field;
5067         struct hist_field *hist_field;
5068         int i, idx = 0;
5069
5070         for_each_hist_field(i, hist_data) {
5071                 hist_field = hist_data->fields[i];
5072                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5073                         tracing_map_cmp_fn_t cmp_fn;
5074
5075                         field = hist_field->field;
5076
5077                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5078                                 cmp_fn = tracing_map_cmp_none;
5079                         else if (!field)
5080                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5081                                                              hist_field->is_signed);
5082                         else if (is_string_field(field))
5083                                 cmp_fn = tracing_map_cmp_string;
5084                         else
5085                                 cmp_fn = tracing_map_cmp_num(field->size,
5086                                                              field->is_signed);
5087                         idx = tracing_map_add_key_field(map,
5088                                                         hist_field->offset,
5089                                                         cmp_fn);
5090                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5091                         idx = tracing_map_add_sum_field(map);
5092
5093                 if (idx < 0)
5094                         return idx;
5095
5096                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5097                         idx = tracing_map_add_var(map);
5098                         if (idx < 0)
5099                                 return idx;
5100                         hist_field->var.idx = idx;
5101                         hist_field->var.hist_data = hist_data;
5102                 }
5103         }
5104
5105         return 0;
5106 }
5107
5108 static struct hist_trigger_data *
5109 create_hist_data(unsigned int map_bits,
5110                  struct hist_trigger_attrs *attrs,
5111                  struct trace_event_file *file,
5112                  bool remove)
5113 {
5114         const struct tracing_map_ops *map_ops = NULL;
5115         struct hist_trigger_data *hist_data;
5116         int ret = 0;
5117
5118         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5119         if (!hist_data)
5120                 return ERR_PTR(-ENOMEM);
5121
5122         hist_data->attrs = attrs;
5123         hist_data->remove = remove;
5124         hist_data->event_file = file;
5125
5126         ret = parse_actions(hist_data);
5127         if (ret)
5128                 goto free;
5129
5130         ret = create_hist_fields(hist_data, file);
5131         if (ret)
5132                 goto free;
5133
5134         ret = create_sort_keys(hist_data);
5135         if (ret)
5136                 goto free;
5137
5138         map_ops = &hist_trigger_elt_data_ops;
5139
5140         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5141                                             map_ops, hist_data);
5142         if (IS_ERR(hist_data->map)) {
5143                 ret = PTR_ERR(hist_data->map);
5144                 hist_data->map = NULL;
5145                 goto free;
5146         }
5147
5148         ret = create_tracing_map_fields(hist_data);
5149         if (ret)
5150                 goto free;
5151  out:
5152         return hist_data;
5153  free:
5154         hist_data->attrs = NULL;
5155
5156         destroy_hist_data(hist_data);
5157
5158         hist_data = ERR_PTR(ret);
5159
5160         goto out;
5161 }
5162
5163 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5164                                     struct tracing_map_elt *elt, void *rec,
5165                                     struct ring_buffer_event *rbe,
5166                                     u64 *var_ref_vals)
5167 {
5168         struct hist_elt_data *elt_data;
5169         struct hist_field *hist_field;
5170         unsigned int i, var_idx;
5171         u64 hist_val;
5172
5173         elt_data = elt->private_data;
5174         elt_data->var_ref_vals = var_ref_vals;
5175
5176         for_each_hist_val_field(i, hist_data) {
5177                 hist_field = hist_data->fields[i];
5178                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5179                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5180                         var_idx = hist_field->var.idx;
5181                         tracing_map_set_var(elt, var_idx, hist_val);
5182                         continue;
5183                 }
5184                 tracing_map_update_sum(elt, i, hist_val);
5185         }
5186
5187         for_each_hist_key_field(i, hist_data) {
5188                 hist_field = hist_data->fields[i];
5189                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5190                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5191                         var_idx = hist_field->var.idx;
5192                         tracing_map_set_var(elt, var_idx, hist_val);
5193                 }
5194         }
5195
5196         update_field_vars(hist_data, elt, rbe, rec);
5197 }
5198
5199 static inline void add_to_key(char *compound_key, void *key,
5200                               struct hist_field *key_field, void *rec)
5201 {
5202         size_t size = key_field->size;
5203
5204         if (key_field->flags & HIST_FIELD_FL_STRING) {
5205                 struct ftrace_event_field *field;
5206
5207                 field = key_field->field;
5208                 if (field->filter_type == FILTER_DYN_STRING)
5209                         size = *(u32 *)(rec + field->offset) >> 16;
5210                 else if (field->filter_type == FILTER_PTR_STRING)
5211                         size = strlen(key);
5212                 else if (field->filter_type == FILTER_STATIC_STRING)
5213                         size = field->size;
5214
5215                 /* ensure NULL-termination */
5216                 if (size > key_field->size - 1)
5217                         size = key_field->size - 1;
5218
5219                 strncpy(compound_key + key_field->offset, (char *)key, size);
5220         } else
5221                 memcpy(compound_key + key_field->offset, key, size);
5222 }
5223
5224 static void
5225 hist_trigger_actions(struct hist_trigger_data *hist_data,
5226                      struct tracing_map_elt *elt, void *rec,
5227                      struct ring_buffer_event *rbe, void *key,
5228                      u64 *var_ref_vals)
5229 {
5230         struct action_data *data;
5231         unsigned int i;
5232
5233         for (i = 0; i < hist_data->n_actions; i++) {
5234                 data = hist_data->actions[i];
5235                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5236         }
5237 }
5238
5239 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5240                                struct ring_buffer_event *rbe)
5241 {
5242         struct hist_trigger_data *hist_data = data->private_data;
5243         bool use_compound_key = (hist_data->n_keys > 1);
5244         unsigned long entries[HIST_STACKTRACE_DEPTH];
5245         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5246         char compound_key[HIST_KEY_SIZE_MAX];
5247         struct tracing_map_elt *elt = NULL;
5248         struct hist_field *key_field;
5249         u64 field_contents;
5250         void *key = NULL;
5251         unsigned int i;
5252
5253         memset(compound_key, 0, hist_data->key_size);
5254
5255         for_each_hist_key_field(i, hist_data) {
5256                 key_field = hist_data->fields[i];
5257
5258                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5259                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5260                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5261                                          HIST_STACKTRACE_SKIP);
5262                         key = entries;
5263                 } else {
5264                         field_contents = key_field->fn(key_field, elt, rbe, rec);
5265                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5266                                 key = (void *)(unsigned long)field_contents;
5267                                 use_compound_key = true;
5268                         } else
5269                                 key = (void *)&field_contents;
5270                 }
5271
5272                 if (use_compound_key)
5273                         add_to_key(compound_key, key, key_field, rec);
5274         }
5275
5276         if (use_compound_key)
5277                 key = compound_key;
5278
5279         if (hist_data->n_var_refs &&
5280             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5281                 return;
5282
5283         elt = tracing_map_insert(hist_data->map, key);
5284         if (!elt)
5285                 return;
5286
5287         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5288
5289         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5290                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5291 }
5292
5293 static void hist_trigger_stacktrace_print(struct seq_file *m,
5294                                           unsigned long *stacktrace_entries,
5295                                           unsigned int max_entries)
5296 {
5297         char str[KSYM_SYMBOL_LEN];
5298         unsigned int spaces = 8;
5299         unsigned int i;
5300
5301         for (i = 0; i < max_entries; i++) {
5302                 if (!stacktrace_entries[i])
5303                         return;
5304
5305                 seq_printf(m, "%*c", 1 + spaces, ' ');
5306                 sprint_symbol(str, stacktrace_entries[i]);
5307                 seq_printf(m, "%s\n", str);
5308         }
5309 }
5310
5311 static void hist_trigger_print_key(struct seq_file *m,
5312                                    struct hist_trigger_data *hist_data,
5313                                    void *key,
5314                                    struct tracing_map_elt *elt)
5315 {
5316         struct hist_field *key_field;
5317         char str[KSYM_SYMBOL_LEN];
5318         bool multiline = false;
5319         const char *field_name;
5320         unsigned int i;
5321         u64 uval;
5322
5323         seq_puts(m, "{ ");
5324
5325         for_each_hist_key_field(i, hist_data) {
5326                 key_field = hist_data->fields[i];
5327
5328                 if (i > hist_data->n_vals)
5329                         seq_puts(m, ", ");
5330
5331                 field_name = hist_field_name(key_field, 0);
5332
5333                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5334                         uval = *(u64 *)(key + key_field->offset);
5335                         seq_printf(m, "%s: %llx", field_name, uval);
5336                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5337                         uval = *(u64 *)(key + key_field->offset);
5338                         sprint_symbol_no_offset(str, uval);
5339                         seq_printf(m, "%s: [%llx] %-45s", field_name,
5340                                    uval, str);
5341                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5342                         uval = *(u64 *)(key + key_field->offset);
5343                         sprint_symbol(str, uval);
5344                         seq_printf(m, "%s: [%llx] %-55s", field_name,
5345                                    uval, str);
5346                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5347                         struct hist_elt_data *elt_data = elt->private_data;
5348                         char *comm;
5349
5350                         if (WARN_ON_ONCE(!elt_data))
5351                                 return;
5352
5353                         comm = elt_data->comm;
5354
5355                         uval = *(u64 *)(key + key_field->offset);
5356                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5357                                    comm, uval);
5358                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5359                         const char *syscall_name;
5360
5361                         uval = *(u64 *)(key + key_field->offset);
5362                         syscall_name = get_syscall_name(uval);
5363                         if (!syscall_name)
5364                                 syscall_name = "unknown_syscall";
5365
5366                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5367                                    syscall_name, uval);
5368                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5369                         seq_puts(m, "stacktrace:\n");
5370                         hist_trigger_stacktrace_print(m,
5371                                                       key + key_field->offset,
5372                                                       HIST_STACKTRACE_DEPTH);
5373                         multiline = true;
5374                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5375                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5376                                    *(u64 *)(key + key_field->offset));
5377                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5378                         seq_printf(m, "%s: %-50s", field_name,
5379                                    (char *)(key + key_field->offset));
5380                 } else {
5381                         uval = *(u64 *)(key + key_field->offset);
5382                         seq_printf(m, "%s: %10llu", field_name, uval);
5383                 }
5384         }
5385
5386         if (!multiline)
5387                 seq_puts(m, " ");
5388
5389         seq_puts(m, "}");
5390 }
5391
5392 static void hist_trigger_entry_print(struct seq_file *m,
5393                                      struct hist_trigger_data *hist_data,
5394                                      void *key,
5395                                      struct tracing_map_elt *elt)
5396 {
5397         const char *field_name;
5398         unsigned int i;
5399
5400         hist_trigger_print_key(m, hist_data, key, elt);
5401
5402         seq_printf(m, " hitcount: %10llu",
5403                    tracing_map_read_sum(elt, HITCOUNT_IDX));
5404
5405         for (i = 1; i < hist_data->n_vals; i++) {
5406                 field_name = hist_field_name(hist_data->fields[i], 0);
5407
5408                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5409                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5410                         continue;
5411
5412                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5413                         seq_printf(m, "  %s: %10llx", field_name,
5414                                    tracing_map_read_sum(elt, i));
5415                 } else {
5416                         seq_printf(m, "  %s: %10llu", field_name,
5417                                    tracing_map_read_sum(elt, i));
5418                 }
5419         }
5420
5421         print_actions(m, hist_data, elt);
5422
5423         seq_puts(m, "\n");
5424 }
5425
5426 static int print_entries(struct seq_file *m,
5427                          struct hist_trigger_data *hist_data)
5428 {
5429         struct tracing_map_sort_entry **sort_entries = NULL;
5430         struct tracing_map *map = hist_data->map;
5431         int i, n_entries;
5432
5433         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5434                                              hist_data->n_sort_keys,
5435                                              &sort_entries);
5436         if (n_entries < 0)
5437                 return n_entries;
5438
5439         for (i = 0; i < n_entries; i++)
5440                 hist_trigger_entry_print(m, hist_data,
5441                                          sort_entries[i]->key,
5442                                          sort_entries[i]->elt);
5443
5444         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5445
5446         return n_entries;
5447 }
5448
5449 static void hist_trigger_show(struct seq_file *m,
5450                               struct event_trigger_data *data, int n)
5451 {
5452         struct hist_trigger_data *hist_data;
5453         int n_entries;
5454
5455         if (n > 0)
5456                 seq_puts(m, "\n\n");
5457
5458         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5459         data->ops->print(m, data->ops, data);
5460         seq_puts(m, "#\n\n");
5461
5462         hist_data = data->private_data;
5463         n_entries = print_entries(m, hist_data);
5464         if (n_entries < 0)
5465                 n_entries = 0;
5466
5467         track_data_snapshot_print(m, hist_data);
5468
5469         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5470                    (u64)atomic64_read(&hist_data->map->hits),
5471                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5472 }
5473
5474 static int hist_show(struct seq_file *m, void *v)
5475 {
5476         struct event_trigger_data *data;
5477         struct trace_event_file *event_file;
5478         int n = 0, ret = 0;
5479
5480         mutex_lock(&event_mutex);
5481
5482         event_file = event_file_data(m->private);
5483         if (unlikely(!event_file)) {
5484                 ret = -ENODEV;
5485                 goto out_unlock;
5486         }
5487
5488         list_for_each_entry_rcu(data, &event_file->triggers, list) {
5489                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5490                         hist_trigger_show(m, data, n++);
5491         }
5492
5493  out_unlock:
5494         mutex_unlock(&event_mutex);
5495
5496         return ret;
5497 }
5498
5499 static int event_hist_open(struct inode *inode, struct file *file)
5500 {
5501         return single_open(file, hist_show, file);
5502 }
5503
5504 const struct file_operations event_hist_fops = {
5505         .open = event_hist_open,
5506         .read = seq_read,
5507         .llseek = seq_lseek,
5508         .release = single_release,
5509 };
5510
5511 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5512 {
5513         const char *field_name = hist_field_name(hist_field, 0);
5514
5515         if (hist_field->var.name)
5516                 seq_printf(m, "%s=", hist_field->var.name);
5517
5518         if (hist_field->flags & HIST_FIELD_FL_CPU)
5519                 seq_puts(m, "cpu");
5520         else if (field_name) {
5521                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5522                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5523                         seq_putc(m, '$');
5524                 seq_printf(m, "%s", field_name);
5525         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5526                 seq_puts(m, "common_timestamp");
5527
5528         if (hist_field->flags) {
5529                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5530                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5531                         const char *flags = get_hist_field_flags(hist_field);
5532
5533                         if (flags)
5534                                 seq_printf(m, ".%s", flags);
5535                 }
5536         }
5537 }
5538
5539 static int event_hist_trigger_print(struct seq_file *m,
5540                                     struct event_trigger_ops *ops,
5541                                     struct event_trigger_data *data)
5542 {
5543         struct hist_trigger_data *hist_data = data->private_data;
5544         struct hist_field *field;
5545         bool have_var = false;
5546         unsigned int i;
5547
5548         seq_puts(m, "hist:");
5549
5550         if (data->name)
5551                 seq_printf(m, "%s:", data->name);
5552
5553         seq_puts(m, "keys=");
5554
5555         for_each_hist_key_field(i, hist_data) {
5556                 field = hist_data->fields[i];
5557
5558                 if (i > hist_data->n_vals)
5559                         seq_puts(m, ",");
5560
5561                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5562                         seq_puts(m, "stacktrace");
5563                 else
5564                         hist_field_print(m, field);
5565         }
5566
5567         seq_puts(m, ":vals=");
5568
5569         for_each_hist_val_field(i, hist_data) {
5570                 field = hist_data->fields[i];
5571                 if (field->flags & HIST_FIELD_FL_VAR) {
5572                         have_var = true;
5573                         continue;
5574                 }
5575
5576                 if (i == HITCOUNT_IDX)
5577                         seq_puts(m, "hitcount");
5578                 else {
5579                         seq_puts(m, ",");
5580                         hist_field_print(m, field);
5581                 }
5582         }
5583
5584         if (have_var) {
5585                 unsigned int n = 0;
5586
5587                 seq_puts(m, ":");
5588
5589                 for_each_hist_val_field(i, hist_data) {
5590                         field = hist_data->fields[i];
5591
5592                         if (field->flags & HIST_FIELD_FL_VAR) {
5593                                 if (n++)
5594                                         seq_puts(m, ",");
5595                                 hist_field_print(m, field);
5596                         }
5597                 }
5598         }
5599
5600         seq_puts(m, ":sort=");
5601
5602         for (i = 0; i < hist_data->n_sort_keys; i++) {
5603                 struct tracing_map_sort_key *sort_key;
5604                 unsigned int idx, first_key_idx;
5605
5606                 /* skip VAR vals */
5607                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5608
5609                 sort_key = &hist_data->sort_keys[i];
5610                 idx = sort_key->field_idx;
5611
5612                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5613                         return -EINVAL;
5614
5615                 if (i > 0)
5616                         seq_puts(m, ",");
5617
5618                 if (idx == HITCOUNT_IDX)
5619                         seq_puts(m, "hitcount");
5620                 else {
5621                         if (idx >= first_key_idx)
5622                                 idx += hist_data->n_vars;
5623                         hist_field_print(m, hist_data->fields[idx]);
5624                 }
5625
5626                 if (sort_key->descending)
5627                         seq_puts(m, ".descending");
5628         }
5629         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5630         if (hist_data->enable_timestamps)
5631                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5632
5633         print_actions_spec(m, hist_data);
5634
5635         if (data->filter_str)
5636                 seq_printf(m, " if %s", data->filter_str);
5637
5638         if (data->paused)
5639                 seq_puts(m, " [paused]");
5640         else
5641                 seq_puts(m, " [active]");
5642
5643         seq_putc(m, '\n');
5644
5645         return 0;
5646 }
5647
5648 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5649                                    struct event_trigger_data *data)
5650 {
5651         struct hist_trigger_data *hist_data = data->private_data;
5652
5653         if (!data->ref && hist_data->attrs->name)
5654                 save_named_trigger(hist_data->attrs->name, data);
5655
5656         data->ref++;
5657
5658         return 0;
5659 }
5660
5661 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5662 {
5663         struct trace_event_file *file;
5664         unsigned int i;
5665         char *cmd;
5666         int ret;
5667
5668         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5669                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5670                 cmd = hist_data->field_var_hists[i]->cmd;
5671                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5672                                               "!hist", "hist", cmd);
5673         }
5674 }
5675
5676 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5677                                     struct event_trigger_data *data)
5678 {
5679         struct hist_trigger_data *hist_data = data->private_data;
5680
5681         if (WARN_ON_ONCE(data->ref <= 0))
5682                 return;
5683
5684         data->ref--;
5685         if (!data->ref) {
5686                 if (data->name)
5687                         del_named_trigger(data);
5688
5689                 trigger_data_free(data);
5690
5691                 remove_hist_vars(hist_data);
5692
5693                 unregister_field_var_hists(hist_data);
5694
5695                 destroy_hist_data(hist_data);
5696         }
5697 }
5698
5699 static struct event_trigger_ops event_hist_trigger_ops = {
5700         .func                   = event_hist_trigger,
5701         .print                  = event_hist_trigger_print,
5702         .init                   = event_hist_trigger_init,
5703         .free                   = event_hist_trigger_free,
5704 };
5705
5706 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5707                                          struct event_trigger_data *data)
5708 {
5709         data->ref++;
5710
5711         save_named_trigger(data->named_data->name, data);
5712
5713         event_hist_trigger_init(ops, data->named_data);
5714
5715         return 0;
5716 }
5717
5718 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5719                                           struct event_trigger_data *data)
5720 {
5721         if (WARN_ON_ONCE(data->ref <= 0))
5722                 return;
5723
5724         event_hist_trigger_free(ops, data->named_data);
5725
5726         data->ref--;
5727         if (!data->ref) {
5728                 del_named_trigger(data);
5729                 trigger_data_free(data);
5730         }
5731 }
5732
5733 static struct event_trigger_ops event_hist_trigger_named_ops = {
5734         .func                   = event_hist_trigger,
5735         .print                  = event_hist_trigger_print,
5736         .init                   = event_hist_trigger_named_init,
5737         .free                   = event_hist_trigger_named_free,
5738 };
5739
5740 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5741                                                             char *param)
5742 {
5743         return &event_hist_trigger_ops;
5744 }
5745
5746 static void hist_clear(struct event_trigger_data *data)
5747 {
5748         struct hist_trigger_data *hist_data = data->private_data;
5749
5750         if (data->name)
5751                 pause_named_trigger(data);
5752
5753         tracepoint_synchronize_unregister();
5754
5755         tracing_map_clear(hist_data->map);
5756
5757         if (data->name)
5758                 unpause_named_trigger(data);
5759 }
5760
5761 static bool compatible_field(struct ftrace_event_field *field,
5762                              struct ftrace_event_field *test_field)
5763 {
5764         if (field == test_field)
5765                 return true;
5766         if (field == NULL || test_field == NULL)
5767                 return false;
5768         if (strcmp(field->name, test_field->name) != 0)
5769                 return false;
5770         if (strcmp(field->type, test_field->type) != 0)
5771                 return false;
5772         if (field->size != test_field->size)
5773                 return false;
5774         if (field->is_signed != test_field->is_signed)
5775                 return false;
5776
5777         return true;
5778 }
5779
5780 static bool hist_trigger_match(struct event_trigger_data *data,
5781                                struct event_trigger_data *data_test,
5782                                struct event_trigger_data *named_data,
5783                                bool ignore_filter)
5784 {
5785         struct tracing_map_sort_key *sort_key, *sort_key_test;
5786         struct hist_trigger_data *hist_data, *hist_data_test;
5787         struct hist_field *key_field, *key_field_test;
5788         unsigned int i;
5789
5790         if (named_data && (named_data != data_test) &&
5791             (named_data != data_test->named_data))
5792                 return false;
5793
5794         if (!named_data && is_named_trigger(data_test))
5795                 return false;
5796
5797         hist_data = data->private_data;
5798         hist_data_test = data_test->private_data;
5799
5800         if (hist_data->n_vals != hist_data_test->n_vals ||
5801             hist_data->n_fields != hist_data_test->n_fields ||
5802             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5803                 return false;
5804
5805         if (!ignore_filter) {
5806                 if ((data->filter_str && !data_test->filter_str) ||
5807                    (!data->filter_str && data_test->filter_str))
5808                         return false;
5809         }
5810
5811         for_each_hist_field(i, hist_data) {
5812                 key_field = hist_data->fields[i];
5813                 key_field_test = hist_data_test->fields[i];
5814
5815                 if (key_field->flags != key_field_test->flags)
5816                         return false;
5817                 if (!compatible_field(key_field->field, key_field_test->field))
5818                         return false;
5819                 if (key_field->offset != key_field_test->offset)
5820                         return false;
5821                 if (key_field->size != key_field_test->size)
5822                         return false;
5823                 if (key_field->is_signed != key_field_test->is_signed)
5824                         return false;
5825                 if (!!key_field->var.name != !!key_field_test->var.name)
5826                         return false;
5827                 if (key_field->var.name &&
5828                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5829                         return false;
5830         }
5831
5832         for (i = 0; i < hist_data->n_sort_keys; i++) {
5833                 sort_key = &hist_data->sort_keys[i];
5834                 sort_key_test = &hist_data_test->sort_keys[i];
5835
5836                 if (sort_key->field_idx != sort_key_test->field_idx ||
5837                     sort_key->descending != sort_key_test->descending)
5838                         return false;
5839         }
5840
5841         if (!ignore_filter && data->filter_str &&
5842             (strcmp(data->filter_str, data_test->filter_str) != 0))
5843                 return false;
5844
5845         if (!actions_match(hist_data, hist_data_test))
5846                 return false;
5847
5848         return true;
5849 }
5850
5851 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5852                                  struct event_trigger_data *data,
5853                                  struct trace_event_file *file)
5854 {
5855         struct hist_trigger_data *hist_data = data->private_data;
5856         struct event_trigger_data *test, *named_data = NULL;
5857         struct trace_array *tr = file->tr;
5858         int ret = 0;
5859
5860         if (hist_data->attrs->name) {
5861                 named_data = find_named_trigger(hist_data->attrs->name);
5862                 if (named_data) {
5863                         if (!hist_trigger_match(data, named_data, named_data,
5864                                                 true)) {
5865                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5866                                 ret = -EINVAL;
5867                                 goto out;
5868                         }
5869                 }
5870         }
5871
5872         if (hist_data->attrs->name && !named_data)
5873                 goto new;
5874
5875         list_for_each_entry_rcu(test, &file->triggers, list) {
5876                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5877                         if (!hist_trigger_match(data, test, named_data, false))
5878                                 continue;
5879                         if (hist_data->attrs->pause)
5880                                 test->paused = true;
5881                         else if (hist_data->attrs->cont)
5882                                 test->paused = false;
5883                         else if (hist_data->attrs->clear)
5884                                 hist_clear(test);
5885                         else {
5886                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5887                                 ret = -EEXIST;
5888                         }
5889                         goto out;
5890                 }
5891         }
5892  new:
5893         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5894                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5895                 ret = -ENOENT;
5896                 goto out;
5897         }
5898
5899         if (hist_data->attrs->pause)
5900                 data->paused = true;
5901
5902         if (named_data) {
5903                 data->private_data = named_data->private_data;
5904                 set_named_trigger_data(data, named_data);
5905                 data->ops = &event_hist_trigger_named_ops;
5906         }
5907
5908         if (data->ops->init) {
5909                 ret = data->ops->init(data->ops, data);
5910                 if (ret < 0)
5911                         goto out;
5912         }
5913
5914         if (hist_data->enable_timestamps) {
5915                 char *clock = hist_data->attrs->clock;
5916
5917                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5918                 if (ret) {
5919                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5920                         goto out;
5921                 }
5922
5923                 tracing_set_time_stamp_abs(file->tr, true);
5924         }
5925
5926         if (named_data)
5927                 destroy_hist_data(hist_data);
5928
5929         ret++;
5930  out:
5931         return ret;
5932 }
5933
5934 static int hist_trigger_enable(struct event_trigger_data *data,
5935                                struct trace_event_file *file)
5936 {
5937         int ret = 0;
5938
5939         list_add_tail_rcu(&data->list, &file->triggers);
5940
5941         update_cond_flag(file);
5942
5943         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5944                 list_del_rcu(&data->list);
5945                 update_cond_flag(file);
5946                 ret--;
5947         }
5948
5949         return ret;
5950 }
5951
5952 static bool have_hist_trigger_match(struct event_trigger_data *data,
5953                                     struct trace_event_file *file)
5954 {
5955         struct hist_trigger_data *hist_data = data->private_data;
5956         struct event_trigger_data *test, *named_data = NULL;
5957         bool match = false;
5958
5959         if (hist_data->attrs->name)
5960                 named_data = find_named_trigger(hist_data->attrs->name);
5961
5962         list_for_each_entry_rcu(test, &file->triggers, list) {
5963                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5964                         if (hist_trigger_match(data, test, named_data, false)) {
5965                                 match = true;
5966                                 break;
5967                         }
5968                 }
5969         }
5970
5971         return match;
5972 }
5973
5974 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5975                                     struct trace_event_file *file)
5976 {
5977         struct hist_trigger_data *hist_data = data->private_data;
5978         struct event_trigger_data *test, *named_data = NULL;
5979
5980         if (hist_data->attrs->name)
5981                 named_data = find_named_trigger(hist_data->attrs->name);
5982
5983         list_for_each_entry_rcu(test, &file->triggers, list) {
5984                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5985                         if (!hist_trigger_match(data, test, named_data, false))
5986                                 continue;
5987                         hist_data = test->private_data;
5988                         if (check_var_refs(hist_data))
5989                                 return true;
5990                         break;
5991                 }
5992         }
5993
5994         return false;
5995 }
5996
5997 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5998                                     struct event_trigger_data *data,
5999                                     struct trace_event_file *file)
6000 {
6001         struct hist_trigger_data *hist_data = data->private_data;
6002         struct event_trigger_data *test, *named_data = NULL;
6003         bool unregistered = false;
6004
6005         if (hist_data->attrs->name)
6006                 named_data = find_named_trigger(hist_data->attrs->name);
6007
6008         list_for_each_entry_rcu(test, &file->triggers, list) {
6009                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6010                         if (!hist_trigger_match(data, test, named_data, false))
6011                                 continue;
6012                         unregistered = true;
6013                         list_del_rcu(&test->list);
6014                         trace_event_trigger_enable_disable(file, 0);
6015                         update_cond_flag(file);
6016                         break;
6017                 }
6018         }
6019
6020         if (unregistered && test->ops->free)
6021                 test->ops->free(test->ops, test);
6022
6023         if (hist_data->enable_timestamps) {
6024                 if (!hist_data->remove || unregistered)
6025                         tracing_set_time_stamp_abs(file->tr, false);
6026         }
6027 }
6028
6029 static bool hist_file_check_refs(struct trace_event_file *file)
6030 {
6031         struct hist_trigger_data *hist_data;
6032         struct event_trigger_data *test;
6033
6034         list_for_each_entry_rcu(test, &file->triggers, list) {
6035                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6036                         hist_data = test->private_data;
6037                         if (check_var_refs(hist_data))
6038                                 return true;
6039                 }
6040         }
6041
6042         return false;
6043 }
6044
6045 static void hist_unreg_all(struct trace_event_file *file)
6046 {
6047         struct event_trigger_data *test, *n;
6048         struct hist_trigger_data *hist_data;
6049         struct synth_event *se;
6050         const char *se_name;
6051
6052         lockdep_assert_held(&event_mutex);
6053
6054         if (hist_file_check_refs(file))
6055                 return;
6056
6057         list_for_each_entry_safe(test, n, &file->triggers, list) {
6058                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6059                         hist_data = test->private_data;
6060                         list_del_rcu(&test->list);
6061                         trace_event_trigger_enable_disable(file, 0);
6062
6063                         se_name = trace_event_name(file->event_call);
6064                         se = find_synth_event(se_name);
6065                         if (se)
6066                                 se->ref--;
6067
6068                         update_cond_flag(file);
6069                         if (hist_data->enable_timestamps)
6070                                 tracing_set_time_stamp_abs(file->tr, false);
6071                         if (test->ops->free)
6072                                 test->ops->free(test->ops, test);
6073                 }
6074         }
6075 }
6076
6077 static int event_hist_trigger_func(struct event_command *cmd_ops,
6078                                    struct trace_event_file *file,
6079                                    char *glob, char *cmd, char *param)
6080 {
6081         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6082         struct event_trigger_data *trigger_data;
6083         struct hist_trigger_attrs *attrs;
6084         struct event_trigger_ops *trigger_ops;
6085         struct hist_trigger_data *hist_data;
6086         struct synth_event *se;
6087         const char *se_name;
6088         bool remove = false;
6089         char *trigger, *p;
6090         int ret = 0;
6091
6092         lockdep_assert_held(&event_mutex);
6093
6094         if (glob && strlen(glob)) {
6095                 hist_err_clear();
6096                 last_cmd_set(file, param);
6097         }
6098
6099         if (!param)
6100                 return -EINVAL;
6101
6102         if (glob[0] == '!')
6103                 remove = true;
6104
6105         /*
6106          * separate the trigger from the filter (k:v [if filter])
6107          * allowing for whitespace in the trigger
6108          */
6109         p = trigger = param;
6110         do {
6111                 p = strstr(p, "if");
6112                 if (!p)
6113                         break;
6114                 if (p == param)
6115                         return -EINVAL;
6116                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6117                         p++;
6118                         continue;
6119                 }
6120                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6121                         return -EINVAL;
6122                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6123                         p++;
6124                         continue;
6125                 }
6126                 break;
6127         } while (p);
6128
6129         if (!p)
6130                 param = NULL;
6131         else {
6132                 *(p - 1) = '\0';
6133                 param = strstrip(p);
6134                 trigger = strstrip(trigger);
6135         }
6136
6137         attrs = parse_hist_trigger_attrs(file->tr, trigger);
6138         if (IS_ERR(attrs))
6139                 return PTR_ERR(attrs);
6140
6141         if (attrs->map_bits)
6142                 hist_trigger_bits = attrs->map_bits;
6143
6144         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6145         if (IS_ERR(hist_data)) {
6146                 destroy_hist_trigger_attrs(attrs);
6147                 return PTR_ERR(hist_data);
6148         }
6149
6150         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6151
6152         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6153         if (!trigger_data) {
6154                 ret = -ENOMEM;
6155                 goto out_free;
6156         }
6157
6158         trigger_data->count = -1;
6159         trigger_data->ops = trigger_ops;
6160         trigger_data->cmd_ops = cmd_ops;
6161
6162         INIT_LIST_HEAD(&trigger_data->list);
6163         RCU_INIT_POINTER(trigger_data->filter, NULL);
6164
6165         trigger_data->private_data = hist_data;
6166
6167         /* if param is non-empty, it's supposed to be a filter */
6168         if (param && cmd_ops->set_filter) {
6169                 ret = cmd_ops->set_filter(param, trigger_data, file);
6170                 if (ret < 0)
6171                         goto out_free;
6172         }
6173
6174         if (remove) {
6175                 if (!have_hist_trigger_match(trigger_data, file))
6176                         goto out_free;
6177
6178                 if (hist_trigger_check_refs(trigger_data, file)) {
6179                         ret = -EBUSY;
6180                         goto out_free;
6181                 }
6182
6183                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6184                 se_name = trace_event_name(file->event_call);
6185                 se = find_synth_event(se_name);
6186                 if (se)
6187                         se->ref--;
6188                 ret = 0;
6189                 goto out_free;
6190         }
6191
6192         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6193         /*
6194          * The above returns on success the # of triggers registered,
6195          * but if it didn't register any it returns zero.  Consider no
6196          * triggers registered a failure too.
6197          */
6198         if (!ret) {
6199                 if (!(attrs->pause || attrs->cont || attrs->clear))
6200                         ret = -ENOENT;
6201                 goto out_free;
6202         } else if (ret < 0)
6203                 goto out_free;
6204
6205         if (get_named_trigger_data(trigger_data))
6206                 goto enable;
6207
6208         if (has_hist_vars(hist_data))
6209                 save_hist_vars(hist_data);
6210
6211         ret = create_actions(hist_data);
6212         if (ret)
6213                 goto out_unreg;
6214
6215         ret = tracing_map_init(hist_data->map);
6216         if (ret)
6217                 goto out_unreg;
6218 enable:
6219         ret = hist_trigger_enable(trigger_data, file);
6220         if (ret)
6221                 goto out_unreg;
6222
6223         se_name = trace_event_name(file->event_call);
6224         se = find_synth_event(se_name);
6225         if (se)
6226                 se->ref++;
6227         /* Just return zero, not the number of registered triggers */
6228         ret = 0;
6229  out:
6230         if (ret == 0)
6231                 hist_err_clear();
6232
6233         return ret;
6234  out_unreg:
6235         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6236  out_free:
6237         if (cmd_ops->set_filter)
6238                 cmd_ops->set_filter(NULL, trigger_data, NULL);
6239
6240         remove_hist_vars(hist_data);
6241
6242         kfree(trigger_data);
6243
6244         destroy_hist_data(hist_data);
6245         goto out;
6246 }
6247
6248 static struct event_command trigger_hist_cmd = {
6249         .name                   = "hist",
6250         .trigger_type           = ETT_EVENT_HIST,
6251         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6252         .func                   = event_hist_trigger_func,
6253         .reg                    = hist_register_trigger,
6254         .unreg                  = hist_unregister_trigger,
6255         .unreg_all              = hist_unreg_all,
6256         .get_trigger_ops        = event_hist_get_trigger_ops,
6257         .set_filter             = set_trigger_filter,
6258 };
6259
6260 __init int register_trigger_hist_cmd(void)
6261 {
6262         int ret;
6263
6264         ret = register_event_command(&trigger_hist_cmd);
6265         WARN_ON(ret < 0);
6266
6267         return ret;
6268 }
6269
6270 static void
6271 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6272                     struct ring_buffer_event *event)
6273 {
6274         struct enable_trigger_data *enable_data = data->private_data;
6275         struct event_trigger_data *test;
6276
6277         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6278                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6279                         if (enable_data->enable)
6280                                 test->paused = false;
6281                         else
6282                                 test->paused = true;
6283                 }
6284         }
6285 }
6286
6287 static void
6288 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6289                           struct ring_buffer_event *event)
6290 {
6291         if (!data->count)
6292                 return;
6293
6294         if (data->count != -1)
6295                 (data->count)--;
6296
6297         hist_enable_trigger(data, rec, event);
6298 }
6299
6300 static struct event_trigger_ops hist_enable_trigger_ops = {
6301         .func                   = hist_enable_trigger,
6302         .print                  = event_enable_trigger_print,
6303         .init                   = event_trigger_init,
6304         .free                   = event_enable_trigger_free,
6305 };
6306
6307 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6308         .func                   = hist_enable_count_trigger,
6309         .print                  = event_enable_trigger_print,
6310         .init                   = event_trigger_init,
6311         .free                   = event_enable_trigger_free,
6312 };
6313
6314 static struct event_trigger_ops hist_disable_trigger_ops = {
6315         .func                   = hist_enable_trigger,
6316         .print                  = event_enable_trigger_print,
6317         .init                   = event_trigger_init,
6318         .free                   = event_enable_trigger_free,
6319 };
6320
6321 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6322         .func                   = hist_enable_count_trigger,
6323         .print                  = event_enable_trigger_print,
6324         .init                   = event_trigger_init,
6325         .free                   = event_enable_trigger_free,
6326 };
6327
6328 static struct event_trigger_ops *
6329 hist_enable_get_trigger_ops(char *cmd, char *param)
6330 {
6331         struct event_trigger_ops *ops;
6332         bool enable;
6333
6334         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6335
6336         if (enable)
6337                 ops = param ? &hist_enable_count_trigger_ops :
6338                         &hist_enable_trigger_ops;
6339         else
6340                 ops = param ? &hist_disable_count_trigger_ops :
6341                         &hist_disable_trigger_ops;
6342
6343         return ops;
6344 }
6345
6346 static void hist_enable_unreg_all(struct trace_event_file *file)
6347 {
6348         struct event_trigger_data *test, *n;
6349
6350         list_for_each_entry_safe(test, n, &file->triggers, list) {
6351                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6352                         list_del_rcu(&test->list);
6353                         update_cond_flag(file);
6354                         trace_event_trigger_enable_disable(file, 0);
6355                         if (test->ops->free)
6356                                 test->ops->free(test->ops, test);
6357                 }
6358         }
6359 }
6360
6361 static struct event_command trigger_hist_enable_cmd = {
6362         .name                   = ENABLE_HIST_STR,
6363         .trigger_type           = ETT_HIST_ENABLE,
6364         .func                   = event_enable_trigger_func,
6365         .reg                    = event_enable_register_trigger,
6366         .unreg                  = event_enable_unregister_trigger,
6367         .unreg_all              = hist_enable_unreg_all,
6368         .get_trigger_ops        = hist_enable_get_trigger_ops,
6369         .set_filter             = set_trigger_filter,
6370 };
6371
6372 static struct event_command trigger_hist_disable_cmd = {
6373         .name                   = DISABLE_HIST_STR,
6374         .trigger_type           = ETT_HIST_ENABLE,
6375         .func                   = event_enable_trigger_func,
6376         .reg                    = event_enable_register_trigger,
6377         .unreg                  = event_enable_unregister_trigger,
6378         .unreg_all              = hist_enable_unreg_all,
6379         .get_trigger_ops        = hist_enable_get_trigger_ops,
6380         .set_filter             = set_trigger_filter,
6381 };
6382
6383 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6384 {
6385         unregister_event_command(&trigger_hist_enable_cmd);
6386         unregister_event_command(&trigger_hist_disable_cmd);
6387 }
6388
6389 __init int register_trigger_hist_enable_disable_cmds(void)
6390 {
6391         int ret;
6392
6393         ret = register_event_command(&trigger_hist_enable_cmd);
6394         if (WARN_ON(ret < 0))
6395                 return ret;
6396         ret = register_event_command(&trigger_hist_disable_cmd);
6397         if (WARN_ON(ret < 0))
6398                 unregister_trigger_hist_enable_disable_cmds();
6399
6400         return ret;
6401 }
6402
6403 static __init int trace_events_hist_init(void)
6404 {
6405         struct dentry *entry = NULL;
6406         struct dentry *d_tracer;
6407         int err = 0;
6408
6409         err = dyn_event_register(&synth_event_ops);
6410         if (err) {
6411                 pr_warn("Could not register synth_event_ops\n");
6412                 return err;
6413         }
6414
6415         d_tracer = tracing_init_dentry();
6416         if (IS_ERR(d_tracer)) {
6417                 err = PTR_ERR(d_tracer);
6418                 goto err;
6419         }
6420
6421         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6422                                     NULL, &synth_events_fops);
6423         if (!entry) {
6424                 err = -ENODEV;
6425                 goto err;
6426         }
6427
6428         return err;
6429  err:
6430         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6431
6432         return err;
6433 }
6434
6435 fs_initcall(trace_events_hist_init);