kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace.h"
  23 #include "trace_dynevent.h"
  24
  25 #define SYNTH_SYSTEM            "synthetic"
  26 #define SYNTH_FIELDS_MAX        32
  27
  28 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  29
  30 #define ERRORS                                                          \
  31         C(NONE,                 "No error"),                            \
  32         C(DUPLICATE_VAR,        "Variable already defined"),            \
  33         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  34         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  35         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  36         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  37         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  38         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  39         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  40         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  41         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  42         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  43         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  44         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  45         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  46         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  47         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  48         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  49         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  50         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  51         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  52         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  53         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  54         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  55         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  56         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  57         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  58         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  59         C(INVALID_PARAM,        "Invalid action param"),                \
  60         C(ACTION_NOT_FOUND,     "No action found"),                     \
  61         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  62         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  63         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  64         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  65         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  66         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  67         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  68         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  69         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  70         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  71         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  72         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  73         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  74         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),
  75
  76 #undef C
  77 #define C(a, b)         HIST_ERR_##a
  78
  79 enum { ERRORS };
  80
  81 #undef C
  82 #define C(a, b)         b
  83
  84 static const char *err_text[] = { ERRORS };
  85
  86 struct hist_field;
  87
  88 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  89                                 struct tracing_map_elt *elt,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96
  97 enum field_op_id {
  98         FIELD_OP_NONE,
  99         FIELD_OP_PLUS,
 100         FIELD_OP_MINUS,
 101         FIELD_OP_UNARY_MINUS,
 102 };
 103
 104 /*
 105  * A hist_var (histogram variable) contains variable information for
 106  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 107  * flag set.  A hist_var has a variable name e.g. ts0, and is
 108  * associated with a given histogram trigger, as specified by
 109  * hist_data.  The hist_var idx is the unique index assigned to the
 110  * variable by the hist trigger's tracing_map.  The idx is what is
 111  * used to set a variable's value and, by a variable reference, to
 112  * retrieve it.
 113  */
 114 struct hist_var {
 115         char                            *name;
 116         struct hist_trigger_data        *hist_data;
 117         unsigned int                    idx;
 118 };
 119
 120 struct hist_field {
 121         struct ftrace_event_field       *field;
 122         unsigned long                   flags;
 123         hist_field_fn_t                 fn;
 124         unsigned int                    ref;
 125         unsigned int                    size;
 126         unsigned int                    offset;
 127         unsigned int                    is_signed;
 128         const char                      *type;
 129         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 130         struct hist_trigger_data        *hist_data;
 131
 132         /*
 133          * Variable fields contain variable-specific info in var.
 134          */
 135         struct hist_var                 var;
 136         enum field_op_id                operator;
 137         char                            *system;
 138         char                            *event_name;
 139
 140         /*
 141          * The name field is used for EXPR and VAR_REF fields.  VAR
 142          * fields contain the variable name in var.name.
 143          */
 144         char                            *name;
 145
 146         /*
 147          * When a histogram trigger is hit, if it has any references
 148          * to variables, the values of those variables are collected
 149          * into a var_ref_vals array by resolve_var_refs().  The
 150          * current value of each variable is read from the tracing_map
 151          * using the hist field's hist_var.idx and entered into the
 152          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 153          */
 154         unsigned int                    var_ref_idx;
 155         bool                            read_once;
 156 };
 157
 158 static u64 hist_field_none(struct hist_field *field,
 159                            struct tracing_map_elt *elt,
 160                            struct ring_buffer_event *rbe,
 161                            void *event)
 162 {
 163         return 0;
 164 }
 165
 166 static u64 hist_field_counter(struct hist_field *field,
 167                               struct tracing_map_elt *elt,
 168                               struct ring_buffer_event *rbe,
 169                               void *event)
 170 {
 171         return 1;
 172 }
 173
 174 static u64 hist_field_string(struct hist_field *hist_field,
 175                              struct tracing_map_elt *elt,
 176                              struct ring_buffer_event *rbe,
 177                              void *event)
 178 {
 179         char *addr = (char *)(event + hist_field->field->offset);
 180
 181         return (u64)(unsigned long)addr;
 182 }
 183
 184 static u64 hist_field_dynstring(struct hist_field *hist_field,
 185                                 struct tracing_map_elt *elt,
 186                                 struct ring_buffer_event *rbe,
 187                                 void *event)
 188 {
 189         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 190         int str_loc = str_item & 0xffff;
 191         char *addr = (char *)(event + str_loc);
 192
 193         return (u64)(unsigned long)addr;
 194 }
 195
 196 static u64 hist_field_pstring(struct hist_field *hist_field,
 197                               struct tracing_map_elt *elt,
 198                               struct ring_buffer_event *rbe,
 199                               void *event)
 200 {
 201         char **addr = (char **)(event + hist_field->field->offset);
 202
 203         return (u64)(unsigned long)*addr;
 204 }
 205
 206 static u64 hist_field_log2(struct hist_field *hist_field,
 207                            struct tracing_map_elt *elt,
 208                            struct ring_buffer_event *rbe,
 209                            void *event)
 210 {
 211         struct hist_field *operand = hist_field->operands[0];
 212
 213         u64 val = operand->fn(operand, elt, rbe, event);
 214
 215         return (u64) ilog2(roundup_pow_of_two(val));
 216 }
 217
 218 static u64 hist_field_plus(struct hist_field *hist_field,
 219                            struct tracing_map_elt *elt,
 220                            struct ring_buffer_event *rbe,
 221                            void *event)
 222 {
 223         struct hist_field *operand1 = hist_field->operands[0];
 224         struct hist_field *operand2 = hist_field->operands[1];
 225
 226         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 227         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 228
 229         return val1 + val2;
 230 }
 231
 232 static u64 hist_field_minus(struct hist_field *hist_field,
 233                             struct tracing_map_elt *elt,
 234                             struct ring_buffer_event *rbe,
 235                             void *event)
 236 {
 237         struct hist_field *operand1 = hist_field->operands[0];
 238         struct hist_field *operand2 = hist_field->operands[1];
 239
 240         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 241         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 242
 243         return val1 - val2;
 244 }
 245
 246 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 247                                   struct tracing_map_elt *elt,
 248                                   struct ring_buffer_event *rbe,
 249                                   void *event)
 250 {
 251         struct hist_field *operand = hist_field->operands[0];
 252
 253         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 254         u64 val = (u64)-sval;
 255
 256         return val;
 257 }
 258
 259 #define DEFINE_HIST_FIELD_FN(type)                                      \
 260         static u64 hist_field_##type(struct hist_field *hist_field,     \
 261                                      struct tracing_map_elt *elt,       \
 262                                      struct ring_buffer_event *rbe,     \
 263                                      void *event)                       \
 264 {                                                                       \
 265         type *addr = (type *)(event + hist_field->field->offset);       \
 266                                                                         \
 267         return (u64)(unsigned long)*addr;                               \
 268 }
 269
 270 DEFINE_HIST_FIELD_FN(s64);
 271 DEFINE_HIST_FIELD_FN(u64);
 272 DEFINE_HIST_FIELD_FN(s32);
 273 DEFINE_HIST_FIELD_FN(u32);
 274 DEFINE_HIST_FIELD_FN(s16);
 275 DEFINE_HIST_FIELD_FN(u16);
 276 DEFINE_HIST_FIELD_FN(s8);
 277 DEFINE_HIST_FIELD_FN(u8);
 278
 279 #define for_each_hist_field(i, hist_data)       \
 280         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 281
 282 #define for_each_hist_val_field(i, hist_data)   \
 283         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 284
 285 #define for_each_hist_key_field(i, hist_data)   \
 286         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 287
 288 #define HIST_STACKTRACE_DEPTH   16
 289 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 290 #define HIST_STACKTRACE_SKIP    5
 291
 292 #define HITCOUNT_IDX            0
 293 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 294
 295 enum hist_field_flags {
 296         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 297         HIST_FIELD_FL_KEY               = 1 << 1,
 298         HIST_FIELD_FL_STRING            = 1 << 2,
 299         HIST_FIELD_FL_HEX               = 1 << 3,
 300         HIST_FIELD_FL_SYM               = 1 << 4,
 301         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 302         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 303         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 304         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 305         HIST_FIELD_FL_LOG2              = 1 << 9,
 306         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 307         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 308         HIST_FIELD_FL_VAR               = 1 << 12,
 309         HIST_FIELD_FL_EXPR              = 1 << 13,
 310         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 311         HIST_FIELD_FL_CPU               = 1 << 15,
 312         HIST_FIELD_FL_ALIAS             = 1 << 16,
 313 };
 314
 315 struct var_defs {
 316         unsigned int    n_vars;
 317         char            *name[TRACING_MAP_VARS_MAX];
 318         char            *expr[TRACING_MAP_VARS_MAX];
 319 };
 320
 321 struct hist_trigger_attrs {
 322         char            *keys_str;
 323         char            *vals_str;
 324         char            *sort_key_str;
 325         char            *name;
 326         char            *clock;
 327         bool            pause;
 328         bool            cont;
 329         bool            clear;
 330         bool            ts_in_usecs;
 331         unsigned int    map_bits;
 332
 333         char            *assignment_str[TRACING_MAP_VARS_MAX];
 334         unsigned int    n_assignments;
 335
 336         char            *action_str[HIST_ACTIONS_MAX];
 337         unsigned int    n_actions;
 338
 339         struct var_defs var_defs;
 340 };
 341
 342 struct field_var {
 343         struct hist_field       *var;
 344         struct hist_field       *val;
 345 };
 346
 347 struct field_var_hist {
 348         struct hist_trigger_data        *hist_data;
 349         char                            *cmd;
 350 };
 351
 352 struct hist_trigger_data {
 353         struct hist_field               *fields[HIST_FIELDS_MAX];
 354         unsigned int                    n_vals;
 355         unsigned int                    n_keys;
 356         unsigned int                    n_fields;
 357         unsigned int                    n_vars;
 358         unsigned int                    key_size;
 359         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 360         unsigned int                    n_sort_keys;
 361         struct trace_event_file         *event_file;
 362         struct hist_trigger_attrs       *attrs;
 363         struct tracing_map              *map;
 364         bool                            enable_timestamps;
 365         bool                            remove;
 366         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 367         unsigned int                    n_var_refs;
 368
 369         struct action_data              *actions[HIST_ACTIONS_MAX];
 370         unsigned int                    n_actions;
 371
 372         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 373         unsigned int                    n_field_vars;
 374         unsigned int                    n_field_var_str;
 375         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 376         unsigned int                    n_field_var_hists;
 377
 378         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 379         unsigned int                    n_save_vars;
 380         unsigned int                    n_save_var_str;
 381 };
 382
 383 static int create_synth_event(int argc, const char **argv);
 384 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
 385 static int synth_event_release(struct dyn_event *ev);
 386 static bool synth_event_is_busy(struct dyn_event *ev);
 387 static bool synth_event_match(const char *system, const char *event,
 388                         int argc, const char **argv, struct dyn_event *ev);
 389
 390 static struct dyn_event_operations synth_event_ops = {
 391         .create = create_synth_event,
 392         .show = synth_event_show,
 393         .is_busy = synth_event_is_busy,
 394         .free = synth_event_release,
 395         .match = synth_event_match,
 396 };
 397
 398 struct synth_field {
 399         char *type;
 400         char *name;
 401         size_t size;
 402         unsigned int offset;
 403         bool is_signed;
 404         bool is_string;
 405 };
 406
 407 struct synth_event {
 408         struct dyn_event                        devent;
 409         int                                     ref;
 410         char                                    *name;
 411         struct synth_field                      **fields;
 412         unsigned int                            n_fields;
 413         unsigned int                            n_u64;
 414         struct trace_event_class                class;
 415         struct trace_event_call                 call;
 416         struct tracepoint                       *tp;
 417         struct module                           *mod;
 418 };
 419
 420 static bool is_synth_event(struct dyn_event *ev)
 421 {
 422         return ev->ops == &synth_event_ops;
 423 }
 424
 425 static struct synth_event *to_synth_event(struct dyn_event *ev)
 426 {
 427         return container_of(ev, struct synth_event, devent);
 428 }
 429
 430 static bool synth_event_is_busy(struct dyn_event *ev)
 431 {
 432         struct synth_event *event = to_synth_event(ev);
 433
 434         return event->ref != 0;
 435 }
 436
 437 static bool synth_event_match(const char *system, const char *event,
 438                         int argc, const char **argv, struct dyn_event *ev)
 439 {
 440         struct synth_event *sev = to_synth_event(ev);
 441
 442         return strcmp(sev->name, event) == 0 &&
 443                 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
 444 }
 445
 446 struct action_data;
 447
 448 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 449                              struct tracing_map_elt *elt, void *rec,
 450                              struct ring_buffer_event *rbe, void *key,
 451                              struct action_data *data, u64 *var_ref_vals);
 452
 453 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 454
 455 enum handler_id {
 456         HANDLER_ONMATCH = 1,
 457         HANDLER_ONMAX,
 458         HANDLER_ONCHANGE,
 459 };
 460
 461 enum action_id {
 462         ACTION_SAVE = 1,
 463         ACTION_TRACE,
 464         ACTION_SNAPSHOT,
 465 };
 466
 467 struct action_data {
 468         enum handler_id         handler;
 469         enum action_id          action;
 470         char                    *action_name;
 471         action_fn_t             fn;
 472
 473         unsigned int            n_params;
 474         char                    *params[SYNTH_FIELDS_MAX];
 475
 476         /*
 477          * When a histogram trigger is hit, the values of any
 478          * references to variables, including variables being passed
 479          * as parameters to synthetic events, are collected into a
 480          * var_ref_vals array.  This var_ref_idx array is an array of
 481          * indices into the var_ref_vals array, one for each synthetic
 482          * event param, and is passed to the synthetic event
 483          * invocation.
 484          */
 485         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 486         struct synth_event      *synth_event;
 487         bool                    use_trace_keyword;
 488         char                    *synth_event_name;
 489
 490         union {
 491                 struct {
 492                         char                    *event;
 493                         char                    *event_system;
 494                 } match_data;
 495
 496                 struct {
 497                         /*
 498                          * var_str contains the $-unstripped variable
 499                          * name referenced by var_ref, and used when
 500                          * printing the action.  Because var_ref
 501                          * creation is deferred to create_actions(),
 502                          * we need a per-action way to save it until
 503                          * then, thus var_str.
 504                          */
 505                         char                    *var_str;
 506
 507                         /*
 508                          * var_ref refers to the variable being
 509                          * tracked e.g onmax($var).
 510                          */
 511                         struct hist_field       *var_ref;
 512
 513                         /*
 514                          * track_var contains the 'invisible' tracking
 515                          * variable created to keep the current
 516                          * e.g. max value.
 517                          */
 518                         struct hist_field       *track_var;
 519
 520                         check_track_val_fn_t    check_val;
 521                         action_fn_t             save_data;
 522                 } track_data;
 523         };
 524 };
 525
 526 struct track_data {
 527         u64                             track_val;
 528         bool                            updated;
 529
 530         unsigned int                    key_len;
 531         void                            *key;
 532         struct tracing_map_elt          elt;
 533
 534         struct action_data              *action_data;
 535         struct hist_trigger_data        *hist_data;
 536 };
 537
 538 struct hist_elt_data {
 539         char *comm;
 540         u64 *var_ref_vals;
 541         char *field_var_str[SYNTH_FIELDS_MAX];
 542 };
 543
 544 struct snapshot_context {
 545         struct tracing_map_elt  *elt;
 546         void                    *key;
 547 };
 548
 549 static void track_data_free(struct track_data *track_data)
 550 {
 551         struct hist_elt_data *elt_data;
 552
 553         if (!track_data)
 554                 return;
 555
 556         kfree(track_data->key);
 557
 558         elt_data = track_data->elt.private_data;
 559         if (elt_data) {
 560                 kfree(elt_data->comm);
 561                 kfree(elt_data);
 562         }
 563
 564         kfree(track_data);
 565 }
 566
 567 static struct track_data *track_data_alloc(unsigned int key_len,
 568                                            struct action_data *action_data,
 569                                            struct hist_trigger_data *hist_data)
 570 {
 571         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 572         struct hist_elt_data *elt_data;
 573
 574         if (!data)
 575                 return ERR_PTR(-ENOMEM);
 576
 577         data->key = kzalloc(key_len, GFP_KERNEL);
 578         if (!data->key) {
 579                 track_data_free(data);
 580                 return ERR_PTR(-ENOMEM);
 581         }
 582
 583         data->key_len = key_len;
 584         data->action_data = action_data;
 585         data->hist_data = hist_data;
 586
 587         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 588         if (!elt_data) {
 589                 track_data_free(data);
 590                 return ERR_PTR(-ENOMEM);
 591         }
 592         data->elt.private_data = elt_data;
 593
 594         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 595         if (!elt_data->comm) {
 596                 track_data_free(data);
 597                 return ERR_PTR(-ENOMEM);
 598         }
 599
 600         return data;
 601 }
 602
 603 static char last_cmd[MAX_FILTER_STR_VAL];
 604 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 605
 606 static int errpos(char *str)
 607 {
 608         return err_pos(last_cmd, str);
 609 }
 610
 611 static void last_cmd_set(struct trace_event_file *file, char *str)
 612 {
 613         const char *system = NULL, *name = NULL;
 614         struct trace_event_call *call;
 615
 616         if (!str)
 617                 return;
 618
 619         strcpy(last_cmd, "hist:");
 620         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 621
 622         if (file) {
 623                 call = file->event_call;
 624
 625                 system = call->class->system;
 626                 if (system) {
 627                         name = trace_event_name(call);
 628                         if (!name)
 629                                 system = NULL;
 630                 }
 631         }
 632
 633         if (system)
 634                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 635 }
 636
 637 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 638 {
 639         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 640                         err_type, err_pos);
 641 }
 642
 643 static void hist_err_clear(void)
 644 {
 645         last_cmd[0] = '\0';
 646         last_cmd_loc[0] = '\0';
 647 }
 648
 649 struct synth_trace_event {
 650         struct trace_entry      ent;
 651         u64                     fields[];
 652 };
 653
 654 static int synth_event_define_fields(struct trace_event_call *call)
 655 {
 656         struct synth_trace_event trace;
 657         int offset = offsetof(typeof(trace), fields);
 658         struct synth_event *event = call->data;
 659         unsigned int i, size, n_u64;
 660         char *name, *type;
 661         bool is_signed;
 662         int ret = 0;
 663
 664         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 665                 size = event->fields[i]->size;
 666                 is_signed = event->fields[i]->is_signed;
 667                 type = event->fields[i]->type;
 668                 name = event->fields[i]->name;
 669                 ret = trace_define_field(call, type, name, offset, size,
 670                                          is_signed, FILTER_OTHER);
 671                 if (ret)
 672                         break;
 673
 674                 event->fields[i]->offset = n_u64;
 675
 676                 if (event->fields[i]->is_string) {
 677                         offset += STR_VAR_LEN_MAX;
 678                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 679                 } else {
 680                         offset += sizeof(u64);
 681                         n_u64++;
 682                 }
 683         }
 684
 685         event->n_u64 = n_u64;
 686
 687         return ret;
 688 }
 689
 690 static bool synth_field_signed(char *type)
 691 {
 692         if (str_has_prefix(type, "u"))
 693                 return false;
 694         if (strcmp(type, "gfp_t") == 0)
 695                 return false;
 696
 697         return true;
 698 }
 699
 700 static int synth_field_is_string(char *type)
 701 {
 702         if (strstr(type, "char[") != NULL)
 703                 return true;
 704
 705         return false;
 706 }
 707
 708 static int synth_field_string_size(char *type)
 709 {
 710         char buf[4], *end, *start;
 711         unsigned int len;
 712         int size, err;
 713
 714         start = strstr(type, "char[");
 715         if (start == NULL)
 716                 return -EINVAL;
 717         start += sizeof("char[") - 1;
 718
 719         end = strchr(type, ']');
 720         if (!end || end < start)
 721                 return -EINVAL;
 722
 723         len = end - start;
 724         if (len > 3)
 725                 return -EINVAL;
 726
 727         strncpy(buf, start, len);
 728         buf[len] = '\0';
 729
 730         err = kstrtouint(buf, 0, &size);
 731         if (err)
 732                 return err;
 733
 734         if (size > STR_VAR_LEN_MAX)
 735                 return -EINVAL;
 736
 737         return size;
 738 }
 739
 740 static int synth_field_size(char *type)
 741 {
 742         int size = 0;
 743
 744         if (strcmp(type, "s64") == 0)
 745                 size = sizeof(s64);
 746         else if (strcmp(type, "u64") == 0)
 747                 size = sizeof(u64);
 748         else if (strcmp(type, "s32") == 0)
 749                 size = sizeof(s32);
 750         else if (strcmp(type, "u32") == 0)
 751                 size = sizeof(u32);
 752         else if (strcmp(type, "s16") == 0)
 753                 size = sizeof(s16);
 754         else if (strcmp(type, "u16") == 0)
 755                 size = sizeof(u16);
 756         else if (strcmp(type, "s8") == 0)
 757                 size = sizeof(s8);
 758         else if (strcmp(type, "u8") == 0)
 759                 size = sizeof(u8);
 760         else if (strcmp(type, "char") == 0)
 761                 size = sizeof(char);
 762         else if (strcmp(type, "unsigned char") == 0)
 763                 size = sizeof(unsigned char);
 764         else if (strcmp(type, "int") == 0)
 765                 size = sizeof(int);
 766         else if (strcmp(type, "unsigned int") == 0)
 767                 size = sizeof(unsigned int);
 768         else if (strcmp(type, "long") == 0)
 769                 size = sizeof(long);
 770         else if (strcmp(type, "unsigned long") == 0)
 771                 size = sizeof(unsigned long);
 772         else if (strcmp(type, "pid_t") == 0)
 773                 size = sizeof(pid_t);
 774         else if (strcmp(type, "gfp_t") == 0)
 775                 size = sizeof(gfp_t);
 776         else if (synth_field_is_string(type))
 777                 size = synth_field_string_size(type);
 778
 779         return size;
 780 }
 781
 782 static const char *synth_field_fmt(char *type)
 783 {
 784         const char *fmt = "%llu";
 785
 786         if (strcmp(type, "s64") == 0)
 787                 fmt = "%lld";
 788         else if (strcmp(type, "u64") == 0)
 789                 fmt = "%llu";
 790         else if (strcmp(type, "s32") == 0)
 791                 fmt = "%d";
 792         else if (strcmp(type, "u32") == 0)
 793                 fmt = "%u";
 794         else if (strcmp(type, "s16") == 0)
 795                 fmt = "%d";
 796         else if (strcmp(type, "u16") == 0)
 797                 fmt = "%u";
 798         else if (strcmp(type, "s8") == 0)
 799                 fmt = "%d";
 800         else if (strcmp(type, "u8") == 0)
 801                 fmt = "%u";
 802         else if (strcmp(type, "char") == 0)
 803                 fmt = "%d";
 804         else if (strcmp(type, "unsigned char") == 0)
 805                 fmt = "%u";
 806         else if (strcmp(type, "int") == 0)
 807                 fmt = "%d";
 808         else if (strcmp(type, "unsigned int") == 0)
 809                 fmt = "%u";
 810         else if (strcmp(type, "long") == 0)
 811                 fmt = "%ld";
 812         else if (strcmp(type, "unsigned long") == 0)
 813                 fmt = "%lu";
 814         else if (strcmp(type, "pid_t") == 0)
 815                 fmt = "%d";
 816         else if (strcmp(type, "gfp_t") == 0)
 817                 fmt = "%x";
 818         else if (synth_field_is_string(type))
 819                 fmt = "%s";
 820
 821         return fmt;
 822 }
 823
 824 static void print_synth_event_num_val(struct trace_seq *s,
 825                                       char *print_fmt, char *name,
 826                                       int size, u64 val, char *space)
 827 {
 828         switch (size) {
 829         case 1:
 830                 trace_seq_printf(s, print_fmt, name, (u8)val, space);
 831                 break;
 832
 833         case 2:
 834                 trace_seq_printf(s, print_fmt, name, (u16)val, space);
 835                 break;
 836
 837         case 4:
 838                 trace_seq_printf(s, print_fmt, name, (u32)val, space);
 839                 break;
 840
 841         default:
 842                 trace_seq_printf(s, print_fmt, name, val, space);
 843                 break;
 844         }
 845 }
 846
 847 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 848                                            int flags,
 849                                            struct trace_event *event)
 850 {
 851         struct trace_array *tr = iter->tr;
 852         struct trace_seq *s = &iter->seq;
 853         struct synth_trace_event *entry;
 854         struct synth_event *se;
 855         unsigned int i, n_u64;
 856         char print_fmt[32];
 857         const char *fmt;
 858
 859         entry = (struct synth_trace_event *)iter->ent;
 860         se = container_of(event, struct synth_event, call.event);
 861
 862         trace_seq_printf(s, "%s: ", se->name);
 863
 864         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 865                 if (trace_seq_has_overflowed(s))
 866                         goto end;
 867
 868                 fmt = synth_field_fmt(se->fields[i]->type);
 869
 870                 /* parameter types */
 871                 if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
 872                         trace_seq_printf(s, "%s ", fmt);
 873
 874                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 875
 876                 /* parameter values */
 877                 if (se->fields[i]->is_string) {
 878                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 879                                          (char *)&entry->fields[n_u64],
 880                                          i == se->n_fields - 1 ? "" : " ");
 881                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 882                 } else {
 883                         struct trace_print_flags __flags[] = {
 884                             __def_gfpflag_names, {-1, NULL} };
 885                         char *space = (i == se->n_fields - 1 ? "" : " ");
 886
 887                         print_synth_event_num_val(s, print_fmt,
 888                                                   se->fields[i]->name,
 889                                                   se->fields[i]->size,
 890                                                   entry->fields[n_u64],
 891                                                   space);
 892
 893                         if (strcmp(se->fields[i]->type, "gfp_t") == 0) {
 894                                 trace_seq_puts(s, " (");
 895                                 trace_print_flags_seq(s, "|",
 896                                                       entry->fields[n_u64],
 897                                                       __flags);
 898                                 trace_seq_putc(s, ')');
 899                         }
 900                         n_u64++;
 901                 }
 902         }
 903 end:
 904         trace_seq_putc(s, '\n');
 905
 906         return trace_handle_return(s);
 907 }
 908
 909 static struct trace_event_functions synth_event_funcs = {
 910         .trace          = print_synth_event
 911 };
 912
 913 static notrace void trace_event_raw_event_synth(void *__data,
 914                                                 u64 *var_ref_vals,
 915                                                 unsigned int *var_ref_idx)
 916 {
 917         struct trace_event_file *trace_file = __data;
 918         struct synth_trace_event *entry;
 919         struct trace_event_buffer fbuffer;
 920         struct trace_buffer *buffer;
 921         struct synth_event *event;
 922         unsigned int i, n_u64, val_idx;
 923         int fields_size = 0;
 924
 925         event = trace_file->event_call->data;
 926
 927         if (trace_trigger_soft_disabled(trace_file))
 928                 return;
 929
 930         fields_size = event->n_u64 * sizeof(u64);
 931
 932         /*
 933          * Avoid ring buffer recursion detection, as this event
 934          * is being performed within another event.
 935          */
 936         buffer = trace_file->tr->array_buffer.buffer;
 937         ring_buffer_nest_start(buffer);
 938
 939         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 940                                            sizeof(*entry) + fields_size);
 941         if (!entry)
 942                 goto out;
 943
 944         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 945                 val_idx = var_ref_idx[i];
 946                 if (event->fields[i]->is_string) {
 947                         char *str_val = (char *)(long)var_ref_vals[val_idx];
 948                         char *str_field = (char *)&entry->fields[n_u64];
 949
 950                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 951                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 952                 } else {
 953                         struct synth_field *field = event->fields[i];
 954                         u64 val = var_ref_vals[val_idx];
 955
 956                         switch (field->size) {
 957                         case 1:
 958                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 959                                 break;
 960
 961                         case 2:
 962                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 963                                 break;
 964
 965                         case 4:
 966                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 967                                 break;
 968
 969                         default:
 970                                 entry->fields[n_u64] = val;
 971                                 break;
 972                         }
 973                         n_u64++;
 974                 }
 975         }
 976
 977         trace_event_buffer_commit(&fbuffer);
 978 out:
 979         ring_buffer_nest_end(buffer);
 980 }
 981
 982 static void free_synth_event_print_fmt(struct trace_event_call *call)
 983 {
 984         if (call) {
 985                 kfree(call->print_fmt);
 986                 call->print_fmt = NULL;
 987         }
 988 }
 989
 990 static int __set_synth_event_print_fmt(struct synth_event *event,
 991                                        char *buf, int len)
 992 {
 993         const char *fmt;
 994         int pos = 0;
 995         int i;
 996
 997         /* When len=0, we just calculate the needed length */
 998 #define LEN_OR_ZERO (len ? len - pos : 0)
 999
1000         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1001         for (i = 0; i < event->n_fields; i++) {
1002                 fmt = synth_field_fmt(event->fields[i]->type);
1003                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
1004                                 event->fields[i]->name, fmt,
1005                                 i == event->n_fields - 1 ? "" : ", ");
1006         }
1007         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1008
1009         for (i = 0; i < event->n_fields; i++) {
1010                 pos += snprintf(buf + pos, LEN_OR_ZERO,
1011                                 ", REC->%s", event->fields[i]->name);
1012         }
1013
1014 #undef LEN_OR_ZERO
1015
1016         /* return the length of print_fmt */
1017         return pos;
1018 }
1019
1020 static int set_synth_event_print_fmt(struct trace_event_call *call)
1021 {
1022         struct synth_event *event = call->data;
1023         char *print_fmt;
1024         int len;
1025
1026         /* First: called with 0 length to calculate the needed length */
1027         len = __set_synth_event_print_fmt(event, NULL, 0);
1028
1029         print_fmt = kmalloc(len + 1, GFP_KERNEL);
1030         if (!print_fmt)
1031                 return -ENOMEM;
1032
1033         /* Second: actually write the @print_fmt */
1034         __set_synth_event_print_fmt(event, print_fmt, len + 1);
1035         call->print_fmt = print_fmt;
1036
1037         return 0;
1038 }
1039
1040 static void free_synth_field(struct synth_field *field)
1041 {
1042         kfree(field->type);
1043         kfree(field->name);
1044         kfree(field);
1045 }
1046
1047 static struct synth_field *parse_synth_field(int argc, const char **argv,
1048                                              int *consumed)
1049 {
1050         struct synth_field *field;
1051         const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
1052         int len, ret = 0;
1053
1054         if (field_type[0] == ';')
1055                 field_type++;
1056
1057         if (!strcmp(field_type, "unsigned")) {
1058                 if (argc < 3)
1059                         return ERR_PTR(-EINVAL);
1060                 prefix = "unsigned ";
1061                 field_type = argv[1];
1062                 field_name = argv[2];
1063                 *consumed = 3;
1064         } else {
1065                 field_name = argv[1];
1066                 *consumed = 2;
1067         }
1068
1069         field = kzalloc(sizeof(*field), GFP_KERNEL);
1070         if (!field)
1071                 return ERR_PTR(-ENOMEM);
1072
1073         len = strlen(field_name);
1074         array = strchr(field_name, '[');
1075         if (array)
1076                 len -= strlen(array);
1077         else if (field_name[len - 1] == ';')
1078                 len--;
1079
1080         field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
1081         if (!field->name) {
1082                 ret = -ENOMEM;
1083                 goto free;
1084         }
1085
1086         if (field_type[0] == ';')
1087                 field_type++;
1088         len = strlen(field_type) + 1;
1089         if (array)
1090                 len += strlen(array);
1091         if (prefix)
1092                 len += strlen(prefix);
1093
1094         field->type = kzalloc(len, GFP_KERNEL);
1095         if (!field->type) {
1096                 ret = -ENOMEM;
1097                 goto free;
1098         }
1099         if (prefix)
1100                 strcat(field->type, prefix);
1101         strcat(field->type, field_type);
1102         if (array) {
1103                 strcat(field->type, array);
1104                 if (field->type[len - 1] == ';')
1105                         field->type[len - 1] = '\0';
1106         }
1107
1108         field->size = synth_field_size(field->type);
1109         if (!field->size) {
1110                 ret = -EINVAL;
1111                 goto free;
1112         }
1113
1114         if (synth_field_is_string(field->type))
1115                 field->is_string = true;
1116
1117         field->is_signed = synth_field_signed(field->type);
1118
1119  out:
1120         return field;
1121  free:
1122         free_synth_field(field);
1123         field = ERR_PTR(ret);
1124         goto out;
1125 }
1126
1127 static void free_synth_tracepoint(struct tracepoint *tp)
1128 {
1129         if (!tp)
1130                 return;
1131
1132         kfree(tp->name);
1133         kfree(tp);
1134 }
1135
1136 static struct tracepoint *alloc_synth_tracepoint(char *name)
1137 {
1138         struct tracepoint *tp;
1139
1140         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1141         if (!tp)
1142                 return ERR_PTR(-ENOMEM);
1143
1144         tp->name = kstrdup(name, GFP_KERNEL);
1145         if (!tp->name) {
1146                 kfree(tp);
1147                 return ERR_PTR(-ENOMEM);
1148         }
1149
1150         return tp;
1151 }
1152
1153 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1154                                     unsigned int *var_ref_idx);
1155
1156 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1157                                unsigned int *var_ref_idx)
1158 {
1159         struct tracepoint *tp = event->tp;
1160
1161         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1162                 struct tracepoint_func *probe_func_ptr;
1163                 synth_probe_func_t probe_func;
1164                 void *__data;
1165
1166                 if (!(cpu_online(raw_smp_processor_id())))
1167                         return;
1168
1169                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1170                 if (probe_func_ptr) {
1171                         do {
1172                                 probe_func = probe_func_ptr->func;
1173                                 __data = probe_func_ptr->data;
1174                                 probe_func(__data, var_ref_vals, var_ref_idx);
1175                         } while ((++probe_func_ptr)->func);
1176                 }
1177         }
1178 }
1179
1180 static struct synth_event *find_synth_event(const char *name)
1181 {
1182         struct dyn_event *pos;
1183         struct synth_event *event;
1184
1185         for_each_dyn_event(pos) {
1186                 if (!is_synth_event(pos))
1187                         continue;
1188                 event = to_synth_event(pos);
1189                 if (strcmp(event->name, name) == 0)
1190                         return event;
1191         }
1192
1193         return NULL;
1194 }
1195
1196 static struct trace_event_fields synth_event_fields_array[] = {
1197         { .type = TRACE_FUNCTION_TYPE,
1198           .define_fields = synth_event_define_fields },
1199         {}
1200 };
1201
1202 static int register_synth_event(struct synth_event *event)
1203 {
1204         struct trace_event_call *call = &event->call;
1205         int ret = 0;
1206
1207         event->call.class = &event->class;
1208         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1209         if (!event->class.system) {
1210                 ret = -ENOMEM;
1211                 goto out;
1212         }
1213
1214         event->tp = alloc_synth_tracepoint(event->name);
1215         if (IS_ERR(event->tp)) {
1216                 ret = PTR_ERR(event->tp);
1217                 event->tp = NULL;
1218                 goto out;
1219         }
1220
1221         INIT_LIST_HEAD(&call->class->fields);
1222         call->event.funcs = &synth_event_funcs;
1223         call->class->fields_array = synth_event_fields_array;
1224
1225         ret = register_trace_event(&call->event);
1226         if (!ret) {
1227                 ret = -ENODEV;
1228                 goto out;
1229         }
1230         call->flags = TRACE_EVENT_FL_TRACEPOINT;
1231         call->class->reg = trace_event_reg;
1232         call->class->probe = trace_event_raw_event_synth;
1233         call->data = event;
1234         call->tp = event->tp;
1235
1236         ret = trace_add_event_call(call);
1237         if (ret) {
1238                 pr_warn("Failed to register synthetic event: %s\n",
1239                         trace_event_name(call));
1240                 goto err;
1241         }
1242
1243         ret = set_synth_event_print_fmt(call);
1244         if (ret < 0) {
1245                 trace_remove_event_call(call);
1246                 goto err;
1247         }
1248  out:
1249         return ret;
1250  err:
1251         unregister_trace_event(&call->event);
1252         goto out;
1253 }
1254
1255 static int unregister_synth_event(struct synth_event *event)
1256 {
1257         struct trace_event_call *call = &event->call;
1258         int ret;
1259
1260         ret = trace_remove_event_call(call);
1261
1262         return ret;
1263 }
1264
1265 static void free_synth_event(struct synth_event *event)
1266 {
1267         unsigned int i;
1268
1269         if (!event)
1270                 return;
1271
1272         for (i = 0; i < event->n_fields; i++)
1273                 free_synth_field(event->fields[i]);
1274
1275         kfree(event->fields);
1276         kfree(event->name);
1277         kfree(event->class.system);
1278         free_synth_tracepoint(event->tp);
1279         free_synth_event_print_fmt(&event->call);
1280         kfree(event);
1281 }
1282
1283 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1284                                              struct synth_field **fields)
1285 {
1286         struct synth_event *event;
1287         unsigned int i;
1288
1289         event = kzalloc(sizeof(*event), GFP_KERNEL);
1290         if (!event) {
1291                 event = ERR_PTR(-ENOMEM);
1292                 goto out;
1293         }
1294
1295         event->name = kstrdup(name, GFP_KERNEL);
1296         if (!event->name) {
1297                 kfree(event);
1298                 event = ERR_PTR(-ENOMEM);
1299                 goto out;
1300         }
1301
1302         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1303         if (!event->fields) {
1304                 free_synth_event(event);
1305                 event = ERR_PTR(-ENOMEM);
1306                 goto out;
1307         }
1308
1309         dyn_event_init(&event->devent, &synth_event_ops);
1310
1311         for (i = 0; i < n_fields; i++)
1312                 event->fields[i] = fields[i];
1313
1314         event->n_fields = n_fields;
1315  out:
1316         return event;
1317 }
1318
1319 static void action_trace(struct hist_trigger_data *hist_data,
1320                          struct tracing_map_elt *elt, void *rec,
1321                          struct ring_buffer_event *rbe, void *key,
1322                          struct action_data *data, u64 *var_ref_vals)
1323 {
1324         struct synth_event *event = data->synth_event;
1325
1326         trace_synth(event, var_ref_vals, data->var_ref_idx);
1327 }
1328
1329 struct hist_var_data {
1330         struct list_head list;
1331         struct hist_trigger_data *hist_data;
1332 };
1333
1334 static int synth_event_check_arg_fn(void *data)
1335 {
1336         struct dynevent_arg_pair *arg_pair = data;
1337         int size;
1338
1339         size = synth_field_size((char *)arg_pair->lhs);
1340
1341         return size ? 0 : -EINVAL;
1342 }
1343
1344 /**
1345  * synth_event_add_field - Add a new field to a synthetic event cmd
1346  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1347  * @type: The type of the new field to add
1348  * @name: The name of the new field to add
1349  *
1350  * Add a new field to a synthetic event cmd object.  Field ordering is in
1351  * the same order the fields are added.
1352  *
1353  * See synth_field_size() for available types. If field_name contains
1354  * [n] the field is considered to be an array.
1355  *
1356  * Return: 0 if successful, error otherwise.
1357  */
1358 int synth_event_add_field(struct dynevent_cmd *cmd, const char *type,
1359                           const char *name)
1360 {
1361         struct dynevent_arg_pair arg_pair;
1362         int ret;
1363
1364         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1365                 return -EINVAL;
1366
1367         if (!type || !name)
1368                 return -EINVAL;
1369
1370         dynevent_arg_pair_init(&arg_pair, 0, ';');
1371
1372         arg_pair.lhs = type;
1373         arg_pair.rhs = name;
1374
1375         ret = dynevent_arg_pair_add(cmd, &arg_pair, synth_event_check_arg_fn);
1376         if (ret)
1377                 return ret;
1378
1379         if (++cmd->n_fields > SYNTH_FIELDS_MAX)
1380                 ret = -EINVAL;
1381
1382         return ret;
1383 }
1384 EXPORT_SYMBOL_GPL(synth_event_add_field);
1385
1386 /**
1387  * synth_event_add_field_str - Add a new field to a synthetic event cmd
1388  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1389  * @type_name: The type and name of the new field to add, as a single string
1390  *
1391  * Add a new field to a synthetic event cmd object, as a single
1392  * string.  The @type_name string is expected to be of the form 'type
1393  * name', which will be appended by ';'.  No sanity checking is done -
1394  * what's passed in is assumed to already be well-formed.  Field
1395  * ordering is in the same order the fields are added.
1396  *
1397  * See synth_field_size() for available types. If field_name contains
1398  * [n] the field is considered to be an array.
1399  *
1400  * Return: 0 if successful, error otherwise.
1401  */
1402 int synth_event_add_field_str(struct dynevent_cmd *cmd, const char *type_name)
1403 {
1404         struct dynevent_arg arg;
1405         int ret;
1406
1407         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1408                 return -EINVAL;
1409
1410         if (!type_name)
1411                 return -EINVAL;
1412
1413         dynevent_arg_init(&arg, ';');
1414
1415         arg.str = type_name;
1416
1417         ret = dynevent_arg_add(cmd, &arg, NULL);
1418         if (ret)
1419                 return ret;
1420
1421         if (++cmd->n_fields > SYNTH_FIELDS_MAX)
1422                 ret = -EINVAL;
1423
1424         return ret;
1425 }
1426 EXPORT_SYMBOL_GPL(synth_event_add_field_str);
1427
1428 /**
1429  * synth_event_add_fields - Add multiple fields to a synthetic event cmd
1430  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1431  * @fields: An array of type/name field descriptions
1432  * @n_fields: The number of field descriptions contained in the fields array
1433  *
1434  * Add a new set of fields to a synthetic event cmd object.  The event
1435  * fields that will be defined for the event should be passed in as an
1436  * array of struct synth_field_desc, and the number of elements in the
1437  * array passed in as n_fields.  Field ordering will retain the
1438  * ordering given in the fields array.
1439  *
1440  * See synth_field_size() for available types. If field_name contains
1441  * [n] the field is considered to be an array.
1442  *
1443  * Return: 0 if successful, error otherwise.
1444  */
1445 int synth_event_add_fields(struct dynevent_cmd *cmd,
1446                            struct synth_field_desc *fields,
1447                            unsigned int n_fields)
1448 {
1449         unsigned int i;
1450         int ret = 0;
1451
1452         for (i = 0; i < n_fields; i++) {
1453                 if (fields[i].type == NULL || fields[i].name == NULL) {
1454                         ret = -EINVAL;
1455                         break;
1456                 }
1457
1458                 ret = synth_event_add_field(cmd, fields[i].type, fields[i].name);
1459                 if (ret)
1460                         break;
1461         }
1462
1463         return ret;
1464 }
1465 EXPORT_SYMBOL_GPL(synth_event_add_fields);
1466
1467 /**
1468  * __synth_event_gen_cmd_start - Start a synthetic event command from arg list
1469  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1470  * @name: The name of the synthetic event
1471  * @mod: The module creating the event, NULL if not created from a module
1472  * @args: Variable number of arg (pairs), one pair for each field
1473  *
1474  * NOTE: Users normally won't want to call this function directly, but
1475  * rather use the synth_event_gen_cmd_start() wrapper, which
1476  * automatically adds a NULL to the end of the arg list.  If this
1477  * function is used directly, make sure the last arg in the variable
1478  * arg list is NULL.
1479  *
1480  * Generate a synthetic event command to be executed by
1481  * synth_event_gen_cmd_end().  This function can be used to generate
1482  * the complete command or only the first part of it; in the latter
1483  * case, synth_event_add_field(), synth_event_add_field_str(), or
1484  * synth_event_add_fields() can be used to add more fields following
1485  * this.
1486  *
1487  * There should be an even number variable args, each pair consisting
1488  * of a type followed by a field name.
1489  *
1490  * See synth_field_size() for available types. If field_name contains
1491  * [n] the field is considered to be an array.
1492  *
1493  * Return: 0 if successful, error otherwise.
1494  */
1495 int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd, const char *name,
1496                                 struct module *mod, ...)
1497 {
1498         struct dynevent_arg arg;
1499         va_list args;
1500         int ret;
1501
1502         cmd->event_name = name;
1503         cmd->private_data = mod;
1504
1505         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1506                 return -EINVAL;
1507
1508         dynevent_arg_init(&arg, 0);
1509         arg.str = name;
1510         ret = dynevent_arg_add(cmd, &arg, NULL);
1511         if (ret)
1512                 return ret;
1513
1514         va_start(args, mod);
1515         for (;;) {
1516                 const char *type, *name;
1517
1518                 type = va_arg(args, const char *);
1519                 if (!type)
1520                         break;
1521                 name = va_arg(args, const char *);
1522                 if (!name)
1523                         break;
1524
1525                 if (++cmd->n_fields > SYNTH_FIELDS_MAX) {
1526                         ret = -EINVAL;
1527                         break;
1528                 }
1529
1530                 ret = synth_event_add_field(cmd, type, name);
1531                 if (ret)
1532                         break;
1533         }
1534         va_end(args);
1535
1536         return ret;
1537 }
1538 EXPORT_SYMBOL_GPL(__synth_event_gen_cmd_start);
1539
1540 /**
1541  * synth_event_gen_cmd_array_start - Start synthetic event command from an array
1542  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1543  * @name: The name of the synthetic event
1544  * @fields: An array of type/name field descriptions
1545  * @n_fields: The number of field descriptions contained in the fields array
1546  *
1547  * Generate a synthetic event command to be executed by
1548  * synth_event_gen_cmd_end().  This function can be used to generate
1549  * the complete command or only the first part of it; in the latter
1550  * case, synth_event_add_field(), synth_event_add_field_str(), or
1551  * synth_event_add_fields() can be used to add more fields following
1552  * this.
1553  *
1554  * The event fields that will be defined for the event should be
1555  * passed in as an array of struct synth_field_desc, and the number of
1556  * elements in the array passed in as n_fields.  Field ordering will
1557  * retain the ordering given in the fields array.
1558  *
1559  * See synth_field_size() for available types. If field_name contains
1560  * [n] the field is considered to be an array.
1561  *
1562  * Return: 0 if successful, error otherwise.
1563  */
1564 int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd, const char *name,
1565                                     struct module *mod,
1566                                     struct synth_field_desc *fields,
1567                                     unsigned int n_fields)
1568 {
1569         struct dynevent_arg arg;
1570         unsigned int i;
1571         int ret = 0;
1572
1573         cmd->event_name = name;
1574         cmd->private_data = mod;
1575
1576         if (cmd->type != DYNEVENT_TYPE_SYNTH)
1577                 return -EINVAL;
1578
1579         if (n_fields > SYNTH_FIELDS_MAX)
1580                 return -EINVAL;
1581
1582         dynevent_arg_init(&arg, 0);
1583         arg.str = name;
1584         ret = dynevent_arg_add(cmd, &arg, NULL);
1585         if (ret)
1586                 return ret;
1587
1588         for (i = 0; i < n_fields; i++) {
1589                 if (fields[i].type == NULL || fields[i].name == NULL)
1590                         return -EINVAL;
1591
1592                 ret = synth_event_add_field(cmd, fields[i].type, fields[i].name);
1593                 if (ret)
1594                         break;
1595         }
1596
1597         return ret;
1598 }
1599 EXPORT_SYMBOL_GPL(synth_event_gen_cmd_array_start);
1600
1601 static int __create_synth_event(int argc, const char *name, const char **argv)
1602 {
1603         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1604         struct synth_event *event = NULL;
1605         int i, consumed = 0, n_fields = 0, ret = 0;
1606
1607         /*
1608          * Argument syntax:
1609          *  - Add synthetic event: <event_name> field[;field] ...
1610          *  - Remove synthetic event: !<event_name> field[;field] ...
1611          *      where 'field' = type field_name
1612          */
1613
1614         if (name[0] == '\0' || argc < 1)
1615                 return -EINVAL;
1616
1617         mutex_lock(&event_mutex);
1618
1619         event = find_synth_event(name);
1620         if (event) {
1621                 ret = -EEXIST;
1622                 goto out;
1623         }
1624
1625         for (i = 0; i < argc - 1; i++) {
1626                 if (strcmp(argv[i], ";") == 0)
1627                         continue;
1628                 if (n_fields == SYNTH_FIELDS_MAX) {
1629                         ret = -EINVAL;
1630                         goto err;
1631                 }
1632
1633                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1634                 if (IS_ERR(field)) {
1635                         ret = PTR_ERR(field);
1636                         goto err;
1637                 }
1638                 fields[n_fields++] = field;
1639                 i += consumed - 1;
1640         }
1641
1642         if (i < argc && strcmp(argv[i], ";") != 0) {
1643                 ret = -EINVAL;
1644                 goto err;
1645         }
1646
1647         event = alloc_synth_event(name, n_fields, fields);
1648         if (IS_ERR(event)) {
1649                 ret = PTR_ERR(event);
1650                 event = NULL;
1651                 goto err;
1652         }
1653         ret = register_synth_event(event);
1654         if (!ret)
1655                 dyn_event_add(&event->devent);
1656         else
1657                 free_synth_event(event);
1658  out:
1659         mutex_unlock(&event_mutex);
1660
1661         return ret;
1662  err:
1663         for (i = 0; i < n_fields; i++)
1664                 free_synth_field(fields[i]);
1665
1666         goto out;
1667 }
1668
1669 /**
1670  * synth_event_create - Create a new synthetic event
1671  * @name: The name of the new sythetic event
1672  * @fields: An array of type/name field descriptions
1673  * @n_fields: The number of field descriptions contained in the fields array
1674  * @mod: The module creating the event, NULL if not created from a module
1675  *
1676  * Create a new synthetic event with the given name under the
1677  * trace/events/synthetic/ directory.  The event fields that will be
1678  * defined for the event should be passed in as an array of struct
1679  * synth_field_desc, and the number elements in the array passed in as
1680  * n_fields. Field ordering will retain the ordering given in the
1681  * fields array.
1682  *
1683  * If the new synthetic event is being created from a module, the mod
1684  * param must be non-NULL.  This will ensure that the trace buffer
1685  * won't contain unreadable events.
1686  *
1687  * The new synth event should be deleted using synth_event_delete()
1688  * function.  The new synthetic event can be generated from modules or
1689  * other kernel code using trace_synth_event() and related functions.
1690  *
1691  * Return: 0 if successful, error otherwise.
1692  */
1693 int synth_event_create(const char *name, struct synth_field_desc *fields,
1694                        unsigned int n_fields, struct module *mod)
1695 {
1696         struct dynevent_cmd cmd;
1697         char *buf;
1698         int ret;
1699
1700         buf = kzalloc(MAX_DYNEVENT_CMD_LEN, GFP_KERNEL);
1701         if (!buf)
1702                 return -ENOMEM;
1703
1704         synth_event_cmd_init(&cmd, buf, MAX_DYNEVENT_CMD_LEN);
1705
1706         ret = synth_event_gen_cmd_array_start(&cmd, name, mod,
1707                                               fields, n_fields);
1708         if (ret)
1709                 goto out;
1710
1711         ret = synth_event_gen_cmd_end(&cmd);
1712  out:
1713         kfree(buf);
1714
1715         return ret;
1716 }
1717 EXPORT_SYMBOL_GPL(synth_event_create);
1718
1719 static int destroy_synth_event(struct synth_event *se)
1720 {
1721         int ret;
1722
1723         if (se->ref)
1724                 ret = -EBUSY;
1725         else {
1726                 ret = unregister_synth_event(se);
1727                 if (!ret) {
1728                         dyn_event_remove(&se->devent);
1729                         free_synth_event(se);
1730                 }
1731         }
1732
1733         return ret;
1734 }
1735
1736 /**
1737  * synth_event_delete - Delete a synthetic event
1738  * @event_name: The name of the new sythetic event
1739  *
1740  * Delete a synthetic event that was created with synth_event_create().
1741  *
1742  * Return: 0 if successful, error otherwise.
1743  */
1744 int synth_event_delete(const char *event_name)
1745 {
1746         struct synth_event *se = NULL;
1747         struct module *mod = NULL;
1748         int ret = -ENOENT;
1749
1750         mutex_lock(&event_mutex);
1751         se = find_synth_event(event_name);
1752         if (se) {
1753                 mod = se->mod;
1754                 ret = destroy_synth_event(se);
1755         }
1756         mutex_unlock(&event_mutex);
1757
1758         if (mod) {
1759                 mutex_lock(&trace_types_lock);
1760                 /*
1761                  * It is safest to reset the ring buffer if the module
1762                  * being unloaded registered any events that were
1763                  * used. The only worry is if a new module gets
1764                  * loaded, and takes on the same id as the events of
1765                  * this module. When printing out the buffer, traced
1766                  * events left over from this module may be passed to
1767                  * the new module events and unexpected results may
1768                  * occur.
1769                  */
1770                 tracing_reset_all_online_cpus();
1771                 mutex_unlock(&trace_types_lock);
1772         }
1773
1774         return ret;
1775 }
1776 EXPORT_SYMBOL_GPL(synth_event_delete);
1777
1778 static int create_or_delete_synth_event(int argc, char **argv)
1779 {
1780         const char *name = argv[0];
1781         int ret;
1782
1783         /* trace_run_command() ensures argc != 0 */
1784         if (name[0] == '!') {
1785                 ret = synth_event_delete(name + 1);
1786                 return ret;
1787         }
1788
1789         ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1790         return ret == -ECANCELED ? -EINVAL : ret;
1791 }
1792
1793 static int synth_event_run_command(struct dynevent_cmd *cmd)
1794 {
1795         struct synth_event *se;
1796         int ret;
1797
1798         ret = trace_run_command(cmd->seq.buffer, create_or_delete_synth_event);
1799         if (ret)
1800                 return ret;
1801
1802         se = find_synth_event(cmd->event_name);
1803         if (WARN_ON(!se))
1804                 return -ENOENT;
1805
1806         se->mod = cmd->private_data;
1807
1808         return ret;
1809 }
1810
1811 /**
1812  * synth_event_cmd_init - Initialize a synthetic event command object
1813  * @cmd: A pointer to the dynevent_cmd struct representing the new event
1814  * @buf: A pointer to the buffer used to build the command
1815  * @maxlen: The length of the buffer passed in @buf
1816  *
1817  * Initialize a synthetic event command object.  Use this before
1818  * calling any of the other dyenvent_cmd functions.
1819  */
1820 void synth_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
1821 {
1822         dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_SYNTH,
1823                           synth_event_run_command);
1824 }
1825 EXPORT_SYMBOL_GPL(synth_event_cmd_init);
1826
1827 static inline int
1828 __synth_event_trace_start(struct trace_event_file *file,
1829                           struct synth_event_trace_state *trace_state)
1830 {
1831         int entry_size, fields_size = 0;
1832         int ret = 0;
1833
1834         memset(trace_state, '\0', sizeof(*trace_state));
1835
1836         /*
1837          * Normal event tracing doesn't get called at all unless the
1838          * ENABLED bit is set (which attaches the probe thus allowing
1839          * this code to be called, etc).  Because this is called
1840          * directly by the user, we don't have that but we still need
1841          * to honor not logging when disabled.  For the the iterated
1842          * trace case, we save the enabed state upon start and just
1843          * ignore the following data calls.
1844          */
1845         if (!(file->flags & EVENT_FILE_FL_ENABLED) ||
1846             trace_trigger_soft_disabled(file)) {
1847                 trace_state->disabled = true;
1848                 ret = -ENOENT;
1849                 goto out;
1850         }
1851
1852         trace_state->event = file->event_call->data;
1853
1854         fields_size = trace_state->event->n_u64 * sizeof(u64);
1855
1856         /*
1857          * Avoid ring buffer recursion detection, as this event
1858          * is being performed within another event.
1859          */
1860         trace_state->buffer = file->tr->array_buffer.buffer;
1861         ring_buffer_nest_start(trace_state->buffer);
1862
1863         entry_size = sizeof(*trace_state->entry) + fields_size;
1864         trace_state->entry = trace_event_buffer_reserve(&trace_state->fbuffer,
1865                                                         file,
1866                                                         entry_size);
1867         if (!trace_state->entry) {
1868                 ring_buffer_nest_end(trace_state->buffer);
1869                 ret = -EINVAL;
1870         }
1871 out:
1872         return ret;
1873 }
1874
1875 static inline void
1876 __synth_event_trace_end(struct synth_event_trace_state *trace_state)
1877 {
1878         trace_event_buffer_commit(&trace_state->fbuffer);
1879
1880         ring_buffer_nest_end(trace_state->buffer);
1881 }
1882
1883 /**
1884  * synth_event_trace - Trace a synthetic event
1885  * @file: The trace_event_file representing the synthetic event
1886  * @n_vals: The number of values in vals
1887  * @args: Variable number of args containing the event values
1888  *
1889  * Trace a synthetic event using the values passed in the variable
1890  * argument list.
1891  *
1892  * The argument list should be a list 'n_vals' u64 values.  The number
1893  * of vals must match the number of field in the synthetic event, and
1894  * must be in the same order as the synthetic event fields.
1895  *
1896  * All vals should be cast to u64, and string vals are just pointers
1897  * to strings, cast to u64.  Strings will be copied into space
1898  * reserved in the event for the string, using these pointers.
1899  *
1900  * Return: 0 on success, err otherwise.
1901  */
1902 int synth_event_trace(struct trace_event_file *file, unsigned int n_vals, ...)
1903 {
1904         struct synth_event_trace_state state;
1905         unsigned int i, n_u64;
1906         va_list args;
1907         int ret;
1908
1909         ret = __synth_event_trace_start(file, &state);
1910         if (ret) {
1911                 if (ret == -ENOENT)
1912                         ret = 0; /* just disabled, not really an error */
1913                 return ret;
1914         }
1915
1916         if (n_vals != state.event->n_fields) {
1917                 ret = -EINVAL;
1918                 goto out;
1919         }
1920
1921         va_start(args, n_vals);
1922         for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) {
1923                 u64 val;
1924
1925                 val = va_arg(args, u64);
1926
1927                 if (state.event->fields[i]->is_string) {
1928                         char *str_val = (char *)(long)val;
1929                         char *str_field = (char *)&state.entry->fields[n_u64];
1930
1931                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
1932                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
1933                 } else {
1934                         struct synth_field *field = state.event->fields[i];
1935
1936                         switch (field->size) {
1937                         case 1:
1938                                 *(u8 *)&state.entry->fields[n_u64] = (u8)val;
1939                                 break;
1940
1941                         case 2:
1942                                 *(u16 *)&state.entry->fields[n_u64] = (u16)val;
1943                                 break;
1944
1945                         case 4:
1946                                 *(u32 *)&state.entry->fields[n_u64] = (u32)val;
1947                                 break;
1948
1949                         default:
1950                                 state.entry->fields[n_u64] = val;
1951                                 break;
1952                         }
1953                         n_u64++;
1954                 }
1955         }
1956         va_end(args);
1957 out:
1958         __synth_event_trace_end(&state);
1959
1960         return ret;
1961 }
1962 EXPORT_SYMBOL_GPL(synth_event_trace);
1963
1964 /**
1965  * synth_event_trace_array - Trace a synthetic event from an array
1966  * @file: The trace_event_file representing the synthetic event
1967  * @vals: Array of values
1968  * @n_vals: The number of values in vals
1969  *
1970  * Trace a synthetic event using the values passed in as 'vals'.
1971  *
1972  * The 'vals' array is just an array of 'n_vals' u64.  The number of
1973  * vals must match the number of field in the synthetic event, and
1974  * must be in the same order as the synthetic event fields.
1975  *
1976  * All vals should be cast to u64, and string vals are just pointers
1977  * to strings, cast to u64.  Strings will be copied into space
1978  * reserved in the event for the string, using these pointers.
1979  *
1980  * Return: 0 on success, err otherwise.
1981  */
1982 int synth_event_trace_array(struct trace_event_file *file, u64 *vals,
1983                             unsigned int n_vals)
1984 {
1985         struct synth_event_trace_state state;
1986         unsigned int i, n_u64;
1987         int ret;
1988
1989         ret = __synth_event_trace_start(file, &state);
1990         if (ret) {
1991                 if (ret == -ENOENT)
1992                         ret = 0; /* just disabled, not really an error */
1993                 return ret;
1994         }
1995
1996         if (n_vals != state.event->n_fields) {
1997                 ret = -EINVAL;
1998                 goto out;
1999         }
2000
2001         for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) {
2002                 if (state.event->fields[i]->is_string) {
2003                         char *str_val = (char *)(long)vals[i];
2004                         char *str_field = (char *)&state.entry->fields[n_u64];
2005
2006                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
2007                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
2008                 } else {
2009                         struct synth_field *field = state.event->fields[i];
2010                         u64 val = vals[i];
2011
2012                         switch (field->size) {
2013                         case 1:
2014                                 *(u8 *)&state.entry->fields[n_u64] = (u8)val;
2015                                 break;
2016
2017                         case 2:
2018                                 *(u16 *)&state.entry->fields[n_u64] = (u16)val;
2019                                 break;
2020
2021                         case 4:
2022                                 *(u32 *)&state.entry->fields[n_u64] = (u32)val;
2023                                 break;
2024
2025                         default:
2026                                 state.entry->fields[n_u64] = val;
2027                                 break;
2028                         }
2029                         n_u64++;
2030                 }
2031         }
2032 out:
2033         __synth_event_trace_end(&state);
2034
2035         return ret;
2036 }
2037 EXPORT_SYMBOL_GPL(synth_event_trace_array);
2038
2039 /**
2040  * synth_event_trace_start - Start piecewise synthetic event trace
2041  * @file: The trace_event_file representing the synthetic event
2042  * @trace_state: A pointer to object tracking the piecewise trace state
2043  *
2044  * Start the trace of a synthetic event field-by-field rather than all
2045  * at once.
2046  *
2047  * This function 'opens' an event trace, which means space is reserved
2048  * for the event in the trace buffer, after which the event's
2049  * individual field values can be set through either
2050  * synth_event_add_next_val() or synth_event_add_val().
2051  *
2052  * A pointer to a trace_state object is passed in, which will keep
2053  * track of the current event trace state until the event trace is
2054  * closed (and the event finally traced) using
2055  * synth_event_trace_end().
2056  *
2057  * Note that synth_event_trace_end() must be called after all values
2058  * have been added for each event trace, regardless of whether adding
2059  * all field values succeeded or not.
2060  *
2061  * Note also that for a given event trace, all fields must be added
2062  * using either synth_event_add_next_val() or synth_event_add_val()
2063  * but not both together or interleaved.
2064  *
2065  * Return: 0 on success, err otherwise.
2066  */
2067 int synth_event_trace_start(struct trace_event_file *file,
2068                             struct synth_event_trace_state *trace_state)
2069 {
2070         int ret;
2071
2072         if (!trace_state)
2073                 return -EINVAL;
2074
2075         ret = __synth_event_trace_start(file, trace_state);
2076         if (ret == -ENOENT)
2077                 ret = 0; /* just disabled, not really an error */
2078
2079         return ret;
2080 }
2081 EXPORT_SYMBOL_GPL(synth_event_trace_start);
2082
2083 static int __synth_event_add_val(const char *field_name, u64 val,
2084                                  struct synth_event_trace_state *trace_state)
2085 {
2086         struct synth_field *field = NULL;
2087         struct synth_trace_event *entry;
2088         struct synth_event *event;
2089         int i, ret = 0;
2090
2091         if (!trace_state) {
2092                 ret = -EINVAL;
2093                 goto out;
2094         }
2095
2096         /* can't mix add_next_synth_val() with add_synth_val() */
2097         if (field_name) {
2098                 if (trace_state->add_next) {
2099                         ret = -EINVAL;
2100                         goto out;
2101                 }
2102                 trace_state->add_name = true;
2103         } else {
2104                 if (trace_state->add_name) {
2105                         ret = -EINVAL;
2106                         goto out;
2107                 }
2108                 trace_state->add_next = true;
2109         }
2110
2111         if (trace_state->disabled)
2112                 goto out;
2113
2114         event = trace_state->event;
2115         if (trace_state->add_name) {
2116                 for (i = 0; i < event->n_fields; i++) {
2117                         field = event->fields[i];
2118                         if (strcmp(field->name, field_name) == 0)
2119                                 break;
2120                 }
2121                 if (!field) {
2122                         ret = -EINVAL;
2123                         goto out;
2124                 }
2125         } else {
2126                 if (trace_state->cur_field >= event->n_fields) {
2127                         ret = -EINVAL;
2128                         goto out;
2129                 }
2130                 field = event->fields[trace_state->cur_field++];
2131         }
2132
2133         entry = trace_state->entry;
2134         if (field->is_string) {
2135                 char *str_val = (char *)(long)val;
2136                 char *str_field;
2137
2138                 if (!str_val) {
2139                         ret = -EINVAL;
2140                         goto out;
2141                 }
2142
2143                 str_field = (char *)&entry->fields[field->offset];
2144                 strscpy(str_field, str_val, STR_VAR_LEN_MAX);
2145         } else {
2146                 switch (field->size) {
2147                 case 1:
2148                         *(u8 *)&trace_state->entry->fields[field->offset] = (u8)val;
2149                         break;
2150
2151                 case 2:
2152                         *(u16 *)&trace_state->entry->fields[field->offset] = (u16)val;
2153                         break;
2154
2155                 case 4:
2156                         *(u32 *)&trace_state->entry->fields[field->offset] = (u32)val;
2157                         break;
2158
2159                 default:
2160                         trace_state->entry->fields[field->offset] = val;
2161                         break;
2162                 }
2163         }
2164  out:
2165         return ret;
2166 }
2167
2168 /**
2169  * synth_event_add_next_val - Add the next field's value to an open synth trace
2170  * @val: The value to set the next field to
2171  * @trace_state: A pointer to object tracking the piecewise trace state
2172  *
2173  * Set the value of the next field in an event that's been opened by
2174  * synth_event_trace_start().
2175  *
2176  * The val param should be the value cast to u64.  If the value points
2177  * to a string, the val param should be a char * cast to u64.
2178  *
2179  * This function assumes all the fields in an event are to be set one
2180  * after another - successive calls to this function are made, one for
2181  * each field, in the order of the fields in the event, until all
2182  * fields have been set.  If you'd rather set each field individually
2183  * without regard to ordering, synth_event_add_val() can be used
2184  * instead.
2185  *
2186  * Note however that synth_event_add_next_val() and
2187  * synth_event_add_val() can't be intermixed for a given event trace -
2188  * one or the other but not both can be used at the same time.
2189  *
2190  * Note also that synth_event_trace_end() must be called after all
2191  * values have been added for each event trace, regardless of whether
2192  * adding all field values succeeded or not.
2193  *
2194  * Return: 0 on success, err otherwise.
2195  */
2196 int synth_event_add_next_val(u64 val,
2197                              struct synth_event_trace_state *trace_state)
2198 {
2199         return __synth_event_add_val(NULL, val, trace_state);
2200 }
2201 EXPORT_SYMBOL_GPL(synth_event_add_next_val);
2202
2203 /**
2204  * synth_event_add_val - Add a named field's value to an open synth trace
2205  * @field_name: The name of the synthetic event field value to set
2206  * @val: The value to set the next field to
2207  * @trace_state: A pointer to object tracking the piecewise trace state
2208  *
2209  * Set the value of the named field in an event that's been opened by
2210  * synth_event_trace_start().
2211  *
2212  * The val param should be the value cast to u64.  If the value points
2213  * to a string, the val param should be a char * cast to u64.
2214  *
2215  * This function looks up the field name, and if found, sets the field
2216  * to the specified value.  This lookup makes this function more
2217  * expensive than synth_event_add_next_val(), so use that or the
2218  * none-piecewise synth_event_trace() instead if efficiency is more
2219  * important.
2220  *
2221  * Note however that synth_event_add_next_val() and
2222  * synth_event_add_val() can't be intermixed for a given event trace -
2223  * one or the other but not both can be used at the same time.
2224  *
2225  * Note also that synth_event_trace_end() must be called after all
2226  * values have been added for each event trace, regardless of whether
2227  * adding all field values succeeded or not.
2228  *
2229  * Return: 0 on success, err otherwise.
2230  */
2231 int synth_event_add_val(const char *field_name, u64 val,
2232                         struct synth_event_trace_state *trace_state)
2233 {
2234         return __synth_event_add_val(field_name, val, trace_state);
2235 }
2236 EXPORT_SYMBOL_GPL(synth_event_add_val);
2237
2238 /**
2239  * synth_event_trace_end - End piecewise synthetic event trace
2240  * @trace_state: A pointer to object tracking the piecewise trace state
2241  *
2242  * End the trace of a synthetic event opened by
2243  * synth_event_trace__start().
2244  *
2245  * This function 'closes' an event trace, which basically means that
2246  * it commits the reserved event and cleans up other loose ends.
2247  *
2248  * A pointer to a trace_state object is passed in, which will keep
2249  * track of the current event trace state opened with
2250  * synth_event_trace_start().
2251  *
2252  * Note that this function must be called after all values have been
2253  * added for each event trace, regardless of whether adding all field
2254  * values succeeded or not.
2255  *
2256  * Return: 0 on success, err otherwise.
2257  */
2258 int synth_event_trace_end(struct synth_event_trace_state *trace_state)
2259 {
2260         if (!trace_state)
2261                 return -EINVAL;
2262
2263         __synth_event_trace_end(trace_state);
2264
2265         return 0;
2266 }
2267 EXPORT_SYMBOL_GPL(synth_event_trace_end);
2268
2269 static int create_synth_event(int argc, const char **argv)
2270 {
2271         const char *name = argv[0];
2272         int len;
2273
2274         if (name[0] != 's' || name[1] != ':')
2275                 return -ECANCELED;
2276         name += 2;
2277
2278         /* This interface accepts group name prefix */
2279         if (strchr(name, '/')) {
2280                 len = str_has_prefix(name, SYNTH_SYSTEM "/");
2281                 if (len == 0)
2282                         return -EINVAL;
2283                 name += len;
2284         }
2285         return __create_synth_event(argc - 1, name, argv + 1);
2286 }
2287
2288 static int synth_event_release(struct dyn_event *ev)
2289 {
2290         struct synth_event *event = to_synth_event(ev);
2291         int ret;
2292
2293         if (event->ref)
2294                 return -EBUSY;
2295
2296         ret = unregister_synth_event(event);
2297         if (ret)
2298                 return ret;
2299
2300         dyn_event_remove(ev);
2301         free_synth_event(event);
2302         return 0;
2303 }
2304
2305 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
2306 {
2307         struct synth_field *field;
2308         unsigned int i;
2309
2310         seq_printf(m, "%s\t", event->name);
2311
2312         for (i = 0; i < event->n_fields; i++) {
2313                 field = event->fields[i];
2314
2315                 /* parameter values */
2316                 seq_printf(m, "%s %s%s", field->type, field->name,
2317                            i == event->n_fields - 1 ? "" : "; ");
2318         }
2319
2320         seq_putc(m, '\n');
2321
2322         return 0;
2323 }
2324
2325 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
2326 {
2327         struct synth_event *event = to_synth_event(ev);
2328
2329         seq_printf(m, "s:%s/", event->class.system);
2330
2331         return __synth_event_show(m, event);
2332 }
2333
2334 static int synth_events_seq_show(struct seq_file *m, void *v)
2335 {
2336         struct dyn_event *ev = v;
2337
2338         if (!is_synth_event(ev))
2339                 return 0;
2340
2341         return __synth_event_show(m, to_synth_event(ev));
2342 }
2343
2344 static const struct seq_operations synth_events_seq_op = {
2345         .start  = dyn_event_seq_start,
2346         .next   = dyn_event_seq_next,
2347         .stop   = dyn_event_seq_stop,
2348         .show   = synth_events_seq_show,
2349 };
2350
2351 static int synth_events_open(struct inode *inode, struct file *file)
2352 {
2353         int ret;
2354
2355         ret = security_locked_down(LOCKDOWN_TRACEFS);
2356         if (ret)
2357                 return ret;
2358
2359         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
2360                 ret = dyn_events_release_all(&synth_event_ops);
2361                 if (ret < 0)
2362                         return ret;
2363         }
2364
2365         return seq_open(file, &synth_events_seq_op);
2366 }
2367
2368 static ssize_t synth_events_write(struct file *file,
2369                                   const char __user *buffer,
2370                                   size_t count, loff_t *ppos)
2371 {
2372         return trace_parse_run_command(file, buffer, count, ppos,
2373                                        create_or_delete_synth_event);
2374 }
2375
2376 static const struct file_operations synth_events_fops = {
2377         .open           = synth_events_open,
2378         .write          = synth_events_write,
2379         .read           = seq_read,
2380         .llseek         = seq_lseek,
2381         .release        = seq_release,
2382 };
2383
2384 static u64 hist_field_timestamp(struct hist_field *hist_field,
2385                                 struct tracing_map_elt *elt,
2386                                 struct ring_buffer_event *rbe,
2387                                 void *event)
2388 {
2389         struct hist_trigger_data *hist_data = hist_field->hist_data;
2390         struct trace_array *tr = hist_data->event_file->tr;
2391
2392         u64 ts = ring_buffer_event_time_stamp(rbe);
2393
2394         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
2395                 ts = ns2usecs(ts);
2396
2397         return ts;
2398 }
2399
2400 static u64 hist_field_cpu(struct hist_field *hist_field,
2401                           struct tracing_map_elt *elt,
2402                           struct ring_buffer_event *rbe,
2403                           void *event)
2404 {
2405         int cpu = smp_processor_id();
2406
2407         return cpu;
2408 }
2409
2410 /**
2411  * check_field_for_var_ref - Check if a VAR_REF field references a variable
2412  * @hist_field: The VAR_REF field to check
2413  * @var_data: The hist trigger that owns the variable
2414  * @var_idx: The trigger variable identifier
2415  *
2416  * Check the given VAR_REF field to see whether or not it references
2417  * the given variable associated with the given trigger.
2418  *
2419  * Return: The VAR_REF field if it does reference the variable, NULL if not
2420  */
2421 static struct hist_field *
2422 check_field_for_var_ref(struct hist_field *hist_field,
2423                         struct hist_trigger_data *var_data,
2424                         unsigned int var_idx)
2425 {
2426         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
2427
2428         if (hist_field && hist_field->var.idx == var_idx &&
2429             hist_field->var.hist_data == var_data)
2430                 return hist_field;
2431
2432         return NULL;
2433 }
2434
2435 /**
2436  * find_var_ref - Check if a trigger has a reference to a trigger variable
2437  * @hist_data: The hist trigger that might have a reference to the variable
2438  * @var_data: The hist trigger that owns the variable
2439  * @var_idx: The trigger variable identifier
2440  *
2441  * Check the list of var_refs[] on the first hist trigger to see
2442  * whether any of them are references to the variable on the second
2443  * trigger.
2444  *
2445  * Return: The VAR_REF field referencing the variable if so, NULL if not
2446  */
2447 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
2448                                        struct hist_trigger_data *var_data,
2449                                        unsigned int var_idx)
2450 {
2451         struct hist_field *hist_field;
2452         unsigned int i;
2453
2454         for (i = 0; i < hist_data->n_var_refs; i++) {
2455                 hist_field = hist_data->var_refs[i];
2456                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
2457                         return hist_field;
2458         }
2459
2460         return NULL;
2461 }
2462
2463 /**
2464  * find_any_var_ref - Check if there is a reference to a given trigger variable
2465  * @hist_data: The hist trigger
2466  * @var_idx: The trigger variable identifier
2467  *
2468  * Check to see whether the given variable is currently referenced by
2469  * any other trigger.
2470  *
2471  * The trigger the variable is defined on is explicitly excluded - the
2472  * assumption being that a self-reference doesn't prevent a trigger
2473  * from being removed.
2474  *
2475  * Return: The VAR_REF field referencing the variable if so, NULL if not
2476  */
2477 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
2478                                            unsigned int var_idx)
2479 {
2480         struct trace_array *tr = hist_data->event_file->tr;
2481         struct hist_field *found = NULL;
2482         struct hist_var_data *var_data;
2483
2484         list_for_each_entry(var_data, &tr->hist_vars, list) {
2485                 if (var_data->hist_data == hist_data)
2486                         continue;
2487                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
2488                 if (found)
2489                         break;
2490         }
2491
2492         return found;
2493 }
2494
2495 /**
2496  * check_var_refs - Check if there is a reference to any of trigger's variables
2497  * @hist_data: The hist trigger
2498  *
2499  * A trigger can define one or more variables.  If any one of them is
2500  * currently referenced by any other trigger, this function will
2501  * determine that.
2502
2503  * Typically used to determine whether or not a trigger can be removed
2504  * - if there are any references to a trigger's variables, it cannot.
2505  *
2506  * Return: True if there is a reference to any of trigger's variables
2507  */
2508 static bool check_var_refs(struct hist_trigger_data *hist_data)
2509 {
2510         struct hist_field *field;
2511         bool found = false;
2512         int i;
2513
2514         for_each_hist_field(i, hist_data) {
2515                 field = hist_data->fields[i];
2516                 if (field && field->flags & HIST_FIELD_FL_VAR) {
2517                         if (find_any_var_ref(hist_data, field->var.idx)) {
2518                                 found = true;
2519                                 break;
2520                         }
2521                 }
2522         }
2523
2524         return found;
2525 }
2526
2527 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
2528 {
2529         struct trace_array *tr = hist_data->event_file->tr;
2530         struct hist_var_data *var_data, *found = NULL;
2531
2532         list_for_each_entry(var_data, &tr->hist_vars, list) {
2533                 if (var_data->hist_data == hist_data) {
2534                         found = var_data;
2535                         break;
2536                 }
2537         }
2538
2539         return found;
2540 }
2541
2542 static bool field_has_hist_vars(struct hist_field *hist_field,
2543                                 unsigned int level)
2544 {
2545         int i;
2546
2547         if (level > 3)
2548                 return false;
2549
2550         if (!hist_field)
2551                 return false;
2552
2553         if (hist_field->flags & HIST_FIELD_FL_VAR ||
2554             hist_field->flags & HIST_FIELD_FL_VAR_REF)
2555                 return true;
2556
2557         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
2558                 struct hist_field *operand;
2559
2560                 operand = hist_field->operands[i];
2561                 if (field_has_hist_vars(operand, level + 1))
2562                         return true;
2563         }
2564
2565         return false;
2566 }
2567
2568 static bool has_hist_vars(struct hist_trigger_data *hist_data)
2569 {
2570         struct hist_field *hist_field;
2571         int i;
2572
2573         for_each_hist_field(i, hist_data) {
2574                 hist_field = hist_data->fields[i];
2575                 if (field_has_hist_vars(hist_field, 0))
2576                         return true;
2577         }
2578
2579         return false;
2580 }
2581
2582 static int save_hist_vars(struct hist_trigger_data *hist_data)
2583 {
2584         struct trace_array *tr = hist_data->event_file->tr;
2585         struct hist_var_data *var_data;
2586
2587         var_data = find_hist_vars(hist_data);
2588         if (var_data)
2589                 return 0;
2590
2591         if (tracing_check_open_get_tr(tr))
2592                 return -ENODEV;
2593
2594         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
2595         if (!var_data) {
2596                 trace_array_put(tr);
2597                 return -ENOMEM;
2598         }
2599
2600         var_data->hist_data = hist_data;
2601         list_add(&var_data->list, &tr->hist_vars);
2602
2603         return 0;
2604 }
2605
2606 static void remove_hist_vars(struct hist_trigger_data *hist_data)
2607 {
2608         struct trace_array *tr = hist_data->event_file->tr;
2609         struct hist_var_data *var_data;
2610
2611         var_data = find_hist_vars(hist_data);
2612         if (!var_data)
2613                 return;
2614
2615         if (WARN_ON(check_var_refs(hist_data)))
2616                 return;
2617
2618         list_del(&var_data->list);
2619
2620         kfree(var_data);
2621
2622         trace_array_put(tr);
2623 }
2624
2625 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
2626                                          const char *var_name)
2627 {
2628         struct hist_field *hist_field, *found = NULL;
2629         int i;
2630
2631         for_each_hist_field(i, hist_data) {
2632                 hist_field = hist_data->fields[i];
2633                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
2634                     strcmp(hist_field->var.name, var_name) == 0) {
2635                         found = hist_field;
2636                         break;
2637                 }
2638         }
2639
2640         return found;
2641 }
2642
2643 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
2644                                    struct trace_event_file *file,
2645                                    const char *var_name)
2646 {
2647         struct hist_trigger_data *test_data;
2648         struct event_trigger_data *test;
2649         struct hist_field *hist_field;
2650
2651         lockdep_assert_held(&event_mutex);
2652
2653         hist_field = find_var_field(hist_data, var_name);
2654         if (hist_field)
2655                 return hist_field;
2656
2657         list_for_each_entry(test, &file->triggers, list) {
2658                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2659                         test_data = test->private_data;
2660                         hist_field = find_var_field(test_data, var_name);
2661                         if (hist_field)
2662                                 return hist_field;
2663                 }
2664         }
2665
2666         return NULL;
2667 }
2668
2669 static struct trace_event_file *find_var_file(struct trace_array *tr,
2670                                               char *system,
2671                                               char *event_name,
2672                                               char *var_name)
2673 {
2674         struct hist_trigger_data *var_hist_data;
2675         struct hist_var_data *var_data;
2676         struct trace_event_file *file, *found = NULL;
2677
2678         if (system)
2679                 return find_event_file(tr, system, event_name);
2680
2681         list_for_each_entry(var_data, &tr->hist_vars, list) {
2682                 var_hist_data = var_data->hist_data;
2683                 file = var_hist_data->event_file;
2684                 if (file == found)
2685                         continue;
2686
2687                 if (find_var_field(var_hist_data, var_name)) {
2688                         if (found) {
2689                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
2690                                 return NULL;
2691                         }
2692
2693                         found = file;
2694                 }
2695         }
2696
2697         return found;
2698 }
2699
2700 static struct hist_field *find_file_var(struct trace_event_file *file,
2701                                         const char *var_name)
2702 {
2703         struct hist_trigger_data *test_data;
2704         struct event_trigger_data *test;
2705         struct hist_field *hist_field;
2706
2707         lockdep_assert_held(&event_mutex);
2708
2709         list_for_each_entry(test, &file->triggers, list) {
2710                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2711                         test_data = test->private_data;
2712                         hist_field = find_var_field(test_data, var_name);
2713                         if (hist_field)
2714                                 return hist_field;
2715                 }
2716         }
2717
2718         return NULL;
2719 }
2720
2721 static struct hist_field *
2722 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
2723 {
2724         struct trace_array *tr = hist_data->event_file->tr;
2725         struct hist_field *hist_field, *found = NULL;
2726         struct trace_event_file *file;
2727         unsigned int i;
2728
2729         for (i = 0; i < hist_data->n_actions; i++) {
2730                 struct action_data *data = hist_data->actions[i];
2731
2732                 if (data->handler == HANDLER_ONMATCH) {
2733                         char *system = data->match_data.event_system;
2734                         char *event_name = data->match_data.event;
2735
2736                         file = find_var_file(tr, system, event_name, var_name);
2737                         if (!file)
2738                                 continue;
2739                         hist_field = find_file_var(file, var_name);
2740                         if (hist_field) {
2741                                 if (found) {
2742                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
2743                                                  errpos(var_name));
2744                                         return ERR_PTR(-EINVAL);
2745                                 }
2746
2747                                 found = hist_field;
2748                         }
2749                 }
2750         }
2751         return found;
2752 }
2753
2754 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
2755                                          char *system,
2756                                          char *event_name,
2757                                          char *var_name)
2758 {
2759         struct trace_array *tr = hist_data->event_file->tr;
2760         struct hist_field *hist_field = NULL;
2761         struct trace_event_file *file;
2762
2763         if (!system || !event_name) {
2764                 hist_field = find_match_var(hist_data, var_name);
2765                 if (IS_ERR(hist_field))
2766                         return NULL;
2767                 if (hist_field)
2768                         return hist_field;
2769         }
2770
2771         file = find_var_file(tr, system, event_name, var_name);
2772         if (!file)
2773                 return NULL;
2774
2775         hist_field = find_file_var(file, var_name);
2776
2777         return hist_field;
2778 }
2779
2780 static u64 hist_field_var_ref(struct hist_field *hist_field,
2781                               struct tracing_map_elt *elt,
2782                               struct ring_buffer_event *rbe,
2783                               void *event)
2784 {
2785         struct hist_elt_data *elt_data;
2786         u64 var_val = 0;
2787
2788         if (WARN_ON_ONCE(!elt))
2789                 return var_val;
2790
2791         elt_data = elt->private_data;
2792         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
2793
2794         return var_val;
2795 }
2796
2797 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
2798                              u64 *var_ref_vals, bool self)
2799 {
2800         struct hist_trigger_data *var_data;
2801         struct tracing_map_elt *var_elt;
2802         struct hist_field *hist_field;
2803         unsigned int i, var_idx;
2804         bool resolved = true;
2805         u64 var_val = 0;
2806
2807         for (i = 0; i < hist_data->n_var_refs; i++) {
2808                 hist_field = hist_data->var_refs[i];
2809                 var_idx = hist_field->var.idx;
2810                 var_data = hist_field->var.hist_data;
2811
2812                 if (var_data == NULL) {
2813                         resolved = false;
2814                         break;
2815                 }
2816
2817                 if ((self && var_data != hist_data) ||
2818                     (!self && var_data == hist_data))
2819                         continue;
2820
2821                 var_elt = tracing_map_lookup(var_data->map, key);
2822                 if (!var_elt) {
2823                         resolved = false;
2824                         break;
2825                 }
2826
2827                 if (!tracing_map_var_set(var_elt, var_idx)) {
2828                         resolved = false;
2829                         break;
2830                 }
2831
2832                 if (self || !hist_field->read_once)
2833                         var_val = tracing_map_read_var(var_elt, var_idx);
2834                 else
2835                         var_val = tracing_map_read_var_once(var_elt, var_idx);
2836
2837                 var_ref_vals[i] = var_val;
2838         }
2839
2840         return resolved;
2841 }
2842
2843 static const char *hist_field_name(struct hist_field *field,
2844                                    unsigned int level)
2845 {
2846         const char *field_name = "";
2847
2848         if (level > 1)
2849                 return field_name;
2850
2851         if (field->field)
2852                 field_name = field->field->name;
2853         else if (field->flags & HIST_FIELD_FL_LOG2 ||
2854                  field->flags & HIST_FIELD_FL_ALIAS)
2855                 field_name = hist_field_name(field->operands[0], ++level);
2856         else if (field->flags & HIST_FIELD_FL_CPU)
2857                 field_name = "cpu";
2858         else if (field->flags & HIST_FIELD_FL_EXPR ||
2859                  field->flags & HIST_FIELD_FL_VAR_REF) {
2860                 if (field->system) {
2861                         static char full_name[MAX_FILTER_STR_VAL];
2862
2863                         strcat(full_name, field->system);
2864                         strcat(full_name, ".");
2865                         strcat(full_name, field->event_name);
2866                         strcat(full_name, ".");
2867                         strcat(full_name, field->name);
2868                         field_name = full_name;
2869                 } else
2870                         field_name = field->name;
2871         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
2872                 field_name = "common_timestamp";
2873
2874         if (field_name == NULL)
2875                 field_name = "";
2876
2877         return field_name;
2878 }
2879
2880 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
2881 {
2882         hist_field_fn_t fn = NULL;
2883
2884         switch (field_size) {
2885         case 8:
2886                 if (field_is_signed)
2887                         fn = hist_field_s64;
2888                 else
2889                         fn = hist_field_u64;
2890                 break;
2891         case 4:
2892                 if (field_is_signed)
2893                         fn = hist_field_s32;
2894                 else
2895                         fn = hist_field_u32;
2896                 break;
2897         case 2:
2898                 if (field_is_signed)
2899                         fn = hist_field_s16;
2900                 else
2901                         fn = hist_field_u16;
2902                 break;
2903         case 1:
2904                 if (field_is_signed)
2905                         fn = hist_field_s8;
2906                 else
2907                         fn = hist_field_u8;
2908                 break;
2909         }
2910
2911         return fn;
2912 }
2913
2914 static int parse_map_size(char *str)
2915 {
2916         unsigned long size, map_bits;
2917         int ret;
2918
2919         ret = kstrtoul(str, 0, &size);
2920         if (ret)
2921                 goto out;
2922
2923         map_bits = ilog2(roundup_pow_of_two(size));
2924         if (map_bits < TRACING_MAP_BITS_MIN ||
2925             map_bits > TRACING_MAP_BITS_MAX)
2926                 ret = -EINVAL;
2927         else
2928                 ret = map_bits;
2929  out:
2930         return ret;
2931 }
2932
2933 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
2934 {
2935         unsigned int i;
2936
2937         if (!attrs)
2938                 return;
2939
2940         for (i = 0; i < attrs->n_assignments; i++)
2941                 kfree(attrs->assignment_str[i]);
2942
2943         for (i = 0; i < attrs->n_actions; i++)
2944                 kfree(attrs->action_str[i]);
2945
2946         kfree(attrs->name);
2947         kfree(attrs->sort_key_str);
2948         kfree(attrs->keys_str);
2949         kfree(attrs->vals_str);
2950         kfree(attrs->clock);
2951         kfree(attrs);
2952 }
2953
2954 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
2955 {
2956         int ret = -EINVAL;
2957
2958         if (attrs->n_actions >= HIST_ACTIONS_MAX)
2959                 return ret;
2960
2961         if ((str_has_prefix(str, "onmatch(")) ||
2962             (str_has_prefix(str, "onmax(")) ||
2963             (str_has_prefix(str, "onchange("))) {
2964                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
2965                 if (!attrs->action_str[attrs->n_actions]) {
2966                         ret = -ENOMEM;
2967                         return ret;
2968                 }
2969                 attrs->n_actions++;
2970                 ret = 0;
2971         }
2972         return ret;
2973 }
2974
2975 static int parse_assignment(struct trace_array *tr,
2976                             char *str, struct hist_trigger_attrs *attrs)
2977 {
2978         int len, ret = 0;
2979
2980         if ((len = str_has_prefix(str, "key=")) ||
2981             (len = str_has_prefix(str, "keys="))) {
2982                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
2983                 if (!attrs->keys_str) {
2984                         ret = -ENOMEM;
2985                         goto out;
2986                 }
2987         } else if ((len = str_has_prefix(str, "val=")) ||
2988                    (len = str_has_prefix(str, "vals=")) ||
2989                    (len = str_has_prefix(str, "values="))) {
2990                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
2991                 if (!attrs->vals_str) {
2992                         ret = -ENOMEM;
2993                         goto out;
2994                 }
2995         } else if ((len = str_has_prefix(str, "sort="))) {
2996                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
2997                 if (!attrs->sort_key_str) {
2998                         ret = -ENOMEM;
2999                         goto out;
3000                 }
3001         } else if (str_has_prefix(str, "name=")) {
3002                 attrs->name = kstrdup(str, GFP_KERNEL);
3003                 if (!attrs->name) {
3004                         ret = -ENOMEM;
3005                         goto out;
3006                 }
3007         } else if ((len = str_has_prefix(str, "clock="))) {
3008                 str += len;
3009
3010                 str = strstrip(str);
3011                 attrs->clock = kstrdup(str, GFP_KERNEL);
3012                 if (!attrs->clock) {
3013                         ret = -ENOMEM;
3014                         goto out;
3015                 }
3016         } else if ((len = str_has_prefix(str, "size="))) {
3017                 int map_bits = parse_map_size(str + len);
3018
3019                 if (map_bits < 0) {
3020                         ret = map_bits;
3021                         goto out;
3022                 }
3023                 attrs->map_bits = map_bits;
3024         } else {
3025                 char *assignment;
3026
3027                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
3028                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
3029                         ret = -EINVAL;
3030                         goto out;
3031                 }
3032
3033                 assignment = kstrdup(str, GFP_KERNEL);
3034                 if (!assignment) {
3035                         ret = -ENOMEM;
3036                         goto out;
3037                 }
3038
3039                 attrs->assignment_str[attrs->n_assignments++] = assignment;
3040         }
3041  out:
3042         return ret;
3043 }
3044
3045 static struct hist_trigger_attrs *
3046 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
3047 {
3048         struct hist_trigger_attrs *attrs;
3049         int ret = 0;
3050
3051         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3052         if (!attrs)
3053                 return ERR_PTR(-ENOMEM);
3054
3055         while (trigger_str) {
3056                 char *str = strsep(&trigger_str, ":");
3057                 char *rhs;
3058
3059                 rhs = strchr(str, '=');
3060                 if (rhs) {
3061                         if (!strlen(++rhs)) {
3062                                 ret = -EINVAL;
3063                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
3064                                 goto free;
3065                         }
3066                         ret = parse_assignment(tr, str, attrs);
3067                         if (ret)
3068                                 goto free;
3069                 } else if (strcmp(str, "pause") == 0)
3070                         attrs->pause = true;
3071                 else if ((strcmp(str, "cont") == 0) ||
3072                          (strcmp(str, "continue") == 0))
3073                         attrs->cont = true;
3074                 else if (strcmp(str, "clear") == 0)
3075                         attrs->clear = true;
3076                 else {
3077                         ret = parse_action(str, attrs);
3078                         if (ret)
3079                                 goto free;
3080                 }
3081         }
3082
3083         if (!attrs->keys_str) {
3084                 ret = -EINVAL;
3085                 goto free;
3086         }
3087
3088         if (!attrs->clock) {
3089                 attrs->clock = kstrdup("global", GFP_KERNEL);
3090                 if (!attrs->clock) {
3091                         ret = -ENOMEM;
3092                         goto free;
3093                 }
3094         }
3095
3096         return attrs;
3097  free:
3098         destroy_hist_trigger_attrs(attrs);
3099
3100         return ERR_PTR(ret);
3101 }
3102
3103 static inline void save_comm(char *comm, struct task_struct *task)
3104 {
3105         if (!task->pid) {
3106                 strcpy(comm, "<idle>");
3107                 return;
3108         }
3109
3110         if (WARN_ON_ONCE(task->pid < 0)) {
3111                 strcpy(comm, "<XXX>");
3112                 return;
3113         }
3114
3115         strncpy(comm, task->comm, TASK_COMM_LEN);
3116 }
3117
3118 static void hist_elt_data_free(struct hist_elt_data *elt_data)
3119 {
3120         unsigned int i;
3121
3122         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
3123                 kfree(elt_data->field_var_str[i]);
3124
3125         kfree(elt_data->comm);
3126         kfree(elt_data);
3127 }
3128
3129 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
3130 {
3131         struct hist_elt_data *elt_data = elt->private_data;
3132
3133         hist_elt_data_free(elt_data);
3134 }
3135
3136 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
3137 {
3138         struct hist_trigger_data *hist_data = elt->map->private_data;
3139         unsigned int size = TASK_COMM_LEN;
3140         struct hist_elt_data *elt_data;
3141         struct hist_field *key_field;
3142         unsigned int i, n_str;
3143
3144         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
3145         if (!elt_data)
3146                 return -ENOMEM;
3147
3148         for_each_hist_key_field(i, hist_data) {
3149                 key_field = hist_data->fields[i];
3150
3151                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
3152                         elt_data->comm = kzalloc(size, GFP_KERNEL);
3153                         if (!elt_data->comm) {
3154                                 kfree(elt_data);
3155                                 return -ENOMEM;
3156                         }
3157                         break;
3158                 }
3159         }
3160
3161         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
3162
3163         size = STR_VAR_LEN_MAX;
3164
3165         for (i = 0; i < n_str; i++) {
3166                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
3167                 if (!elt_data->field_var_str[i]) {
3168                         hist_elt_data_free(elt_data);
3169                         return -ENOMEM;
3170                 }
3171         }
3172
3173         elt->private_data = elt_data;
3174
3175         return 0;
3176 }
3177
3178 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
3179 {
3180         struct hist_elt_data *elt_data = elt->private_data;
3181
3182         if (elt_data->comm)
3183                 save_comm(elt_data->comm, current);
3184 }
3185
3186 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
3187         .elt_alloc      = hist_trigger_elt_data_alloc,
3188         .elt_free       = hist_trigger_elt_data_free,
3189         .elt_init       = hist_trigger_elt_data_init,
3190 };
3191
3192 static const char *get_hist_field_flags(struct hist_field *hist_field)
3193 {
3194         const char *flags_str = NULL;
3195
3196         if (hist_field->flags & HIST_FIELD_FL_HEX)
3197                 flags_str = "hex";
3198         else if (hist_field->flags & HIST_FIELD_FL_SYM)
3199                 flags_str = "sym";
3200         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
3201                 flags_str = "sym-offset";
3202         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
3203                 flags_str = "execname";
3204         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
3205                 flags_str = "syscall";
3206         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
3207                 flags_str = "log2";
3208         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
3209                 flags_str = "usecs";
3210
3211         return flags_str;
3212 }
3213
3214 static void expr_field_str(struct hist_field *field, char *expr)
3215 {
3216         if (field->flags & HIST_FIELD_FL_VAR_REF)
3217                 strcat(expr, "$");
3218
3219         strcat(expr, hist_field_name(field, 0));
3220
3221         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
3222                 const char *flags_str = get_hist_field_flags(field);
3223
3224                 if (flags_str) {
3225                         strcat(expr, ".");
3226                         strcat(expr, flags_str);
3227                 }
3228         }
3229 }
3230
3231 static char *expr_str(struct hist_field *field, unsigned int level)
3232 {
3233         char *expr;
3234
3235         if (level > 1)
3236                 return NULL;
3237
3238         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3239         if (!expr)
3240                 return NULL;
3241
3242         if (!field->operands[0]) {
3243                 expr_field_str(field, expr);
3244                 return expr;
3245         }
3246
3247         if (field->operator == FIELD_OP_UNARY_MINUS) {
3248                 char *subexpr;
3249
3250                 strcat(expr, "-(");
3251                 subexpr = expr_str(field->operands[0], ++level);
3252                 if (!subexpr) {
3253                         kfree(expr);
3254                         return NULL;
3255                 }
3256                 strcat(expr, subexpr);
3257                 strcat(expr, ")");
3258
3259                 kfree(subexpr);
3260
3261                 return expr;
3262         }
3263
3264         expr_field_str(field->operands[0], expr);
3265
3266         switch (field->operator) {
3267         case FIELD_OP_MINUS:
3268                 strcat(expr, "-");
3269                 break;
3270         case FIELD_OP_PLUS:
3271                 strcat(expr, "+");
3272                 break;
3273         default:
3274                 kfree(expr);
3275                 return NULL;
3276         }
3277
3278         expr_field_str(field->operands[1], expr);
3279
3280         return expr;
3281 }
3282
3283 static int contains_operator(char *str)
3284 {
3285         enum field_op_id field_op = FIELD_OP_NONE;
3286         char *op;
3287
3288         op = strpbrk(str, "+-");
3289         if (!op)
3290                 return FIELD_OP_NONE;
3291
3292         switch (*op) {
3293         case '-':
3294                 if (*str == '-')
3295                         field_op = FIELD_OP_UNARY_MINUS;
3296                 else
3297                         field_op = FIELD_OP_MINUS;
3298                 break;
3299         case '+':
3300                 field_op = FIELD_OP_PLUS;
3301                 break;
3302         default:
3303                 break;
3304         }
3305
3306         return field_op;
3307 }
3308
3309 static void get_hist_field(struct hist_field *hist_field)
3310 {
3311         hist_field->ref++;
3312 }
3313
3314 static void __destroy_hist_field(struct hist_field *hist_field)
3315 {
3316         if (--hist_field->ref > 1)
3317                 return;
3318
3319         kfree(hist_field->var.name);
3320         kfree(hist_field->name);
3321         kfree(hist_field->type);
3322
3323         kfree(hist_field->system);
3324         kfree(hist_field->event_name);
3325
3326         kfree(hist_field);
3327 }
3328
3329 static void destroy_hist_field(struct hist_field *hist_field,
3330                                unsigned int level)
3331 {
3332         unsigned int i;
3333
3334         if (level > 3)
3335                 return;
3336
3337         if (!hist_field)
3338                 return;
3339
3340         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
3341                 return; /* var refs will be destroyed separately */
3342
3343         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
3344                 destroy_hist_field(hist_field->operands[i], level + 1);
3345
3346         __destroy_hist_field(hist_field);
3347 }
3348
3349 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
3350                                             struct ftrace_event_field *field,
3351                                             unsigned long flags,
3352                                             char *var_name)
3353 {
3354         struct hist_field *hist_field;
3355
3356         if (field && is_function_field(field))
3357                 return NULL;
3358
3359         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3360         if (!hist_field)
3361                 return NULL;
3362
3363         hist_field->ref = 1;
3364
3365         hist_field->hist_data = hist_data;
3366
3367         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
3368                 goto out; /* caller will populate */
3369
3370         if (flags & HIST_FIELD_FL_VAR_REF) {
3371                 hist_field->fn = hist_field_var_ref;
3372                 goto out;
3373         }
3374
3375         if (flags & HIST_FIELD_FL_HITCOUNT) {
3376                 hist_field->fn = hist_field_counter;
3377                 hist_field->size = sizeof(u64);
3378                 hist_field->type = kstrdup("u64", GFP_KERNEL);
3379                 if (!hist_field->type)
3380                         goto free;
3381                 goto out;
3382         }
3383
3384         if (flags & HIST_FIELD_FL_STACKTRACE) {
3385                 hist_field->fn = hist_field_none;
3386                 goto out;
3387         }
3388
3389         if (flags & HIST_FIELD_FL_LOG2) {
3390                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
3391                 hist_field->fn = hist_field_log2;
3392                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
3393                 hist_field->size = hist_field->operands[0]->size;
3394                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
3395                 if (!hist_field->type)
3396                         goto free;
3397                 goto out;
3398         }
3399
3400         if (flags & HIST_FIELD_FL_TIMESTAMP) {
3401                 hist_field->fn = hist_field_timestamp;
3402                 hist_field->size = sizeof(u64);
3403                 hist_field->type = kstrdup("u64", GFP_KERNEL);
3404                 if (!hist_field->type)
3405                         goto free;
3406                 goto out;
3407         }
3408
3409         if (flags & HIST_FIELD_FL_CPU) {
3410                 hist_field->fn = hist_field_cpu;
3411                 hist_field->size = sizeof(int);
3412                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
3413                 if (!hist_field->type)
3414                         goto free;
3415                 goto out;
3416         }
3417
3418         if (WARN_ON_ONCE(!field))
3419                 goto out;
3420
3421         if (is_string_field(field)) {
3422                 flags |= HIST_FIELD_FL_STRING;
3423
3424                 hist_field->size = MAX_FILTER_STR_VAL;
3425                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
3426                 if (!hist_field->type)
3427                         goto free;
3428
3429                 if (field->filter_type == FILTER_STATIC_STRING)
3430                         hist_field->fn = hist_field_string;
3431                 else if (field->filter_type == FILTER_DYN_STRING)
3432                         hist_field->fn = hist_field_dynstring;
3433                 else
3434                         hist_field->fn = hist_field_pstring;
3435         } else {
3436                 hist_field->size = field->size;
3437                 hist_field->is_signed = field->is_signed;
3438                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
3439                 if (!hist_field->type)
3440                         goto free;
3441
3442                 hist_field->fn = select_value_fn(field->size,
3443                                                  field->is_signed);
3444                 if (!hist_field->fn) {
3445                         destroy_hist_field(hist_field, 0);
3446                         return NULL;
3447                 }
3448         }
3449  out:
3450         hist_field->field = field;
3451         hist_field->flags = flags;
3452
3453         if (var_name) {
3454                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
3455                 if (!hist_field->var.name)
3456                         goto free;
3457         }
3458
3459         return hist_field;
3460  free:
3461         destroy_hist_field(hist_field, 0);
3462         return NULL;
3463 }
3464
3465 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
3466 {
3467         unsigned int i;
3468
3469         for (i = 0; i < HIST_FIELDS_MAX; i++) {
3470                 if (hist_data->fields[i]) {
3471                         destroy_hist_field(hist_data->fields[i], 0);
3472                         hist_data->fields[i] = NULL;
3473                 }
3474         }
3475
3476         for (i = 0; i < hist_data->n_var_refs; i++) {
3477                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
3478                 __destroy_hist_field(hist_data->var_refs[i]);
3479                 hist_data->var_refs[i] = NULL;
3480         }
3481 }
3482
3483 static int init_var_ref(struct hist_field *ref_field,
3484                         struct hist_field *var_field,
3485                         char *system, char *event_name)
3486 {
3487         int err = 0;
3488
3489         ref_field->var.idx = var_field->var.idx;
3490         ref_field->var.hist_data = var_field->hist_data;
3491         ref_field->size = var_field->size;
3492         ref_field->is_signed = var_field->is_signed;
3493         ref_field->flags |= var_field->flags &
3494                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3495
3496         if (system) {
3497                 ref_field->system = kstrdup(system, GFP_KERNEL);
3498                 if (!ref_field->system)
3499                         return -ENOMEM;
3500         }
3501
3502         if (event_name) {
3503                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
3504                 if (!ref_field->event_name) {
3505                         err = -ENOMEM;
3506                         goto free;
3507                 }
3508         }
3509
3510         if (var_field->var.name) {
3511                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
3512                 if (!ref_field->name) {
3513                         err = -ENOMEM;
3514                         goto free;
3515                 }
3516         } else if (var_field->name) {
3517                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
3518                 if (!ref_field->name) {
3519                         err = -ENOMEM;
3520                         goto free;
3521                 }
3522         }
3523
3524         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
3525         if (!ref_field->type) {
3526                 err = -ENOMEM;
3527                 goto free;
3528         }
3529  out:
3530         return err;
3531  free:
3532         kfree(ref_field->system);
3533         kfree(ref_field->event_name);
3534         kfree(ref_field->name);
3535
3536         goto out;
3537 }
3538
3539 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
3540                             struct hist_field *var_field)
3541 {
3542         struct hist_field *ref_field;
3543         int i;
3544
3545         for (i = 0; i < hist_data->n_var_refs; i++) {
3546                 ref_field = hist_data->var_refs[i];
3547                 if (ref_field->var.idx == var_field->var.idx &&
3548                     ref_field->var.hist_data == var_field->hist_data)
3549                         return i;
3550         }
3551
3552         return -ENOENT;
3553 }
3554
3555 /**
3556  * create_var_ref - Create a variable reference and attach it to trigger
3557  * @hist_data: The trigger that will be referencing the variable
3558  * @var_field: The VAR field to create a reference to
3559  * @system: The optional system string
3560  * @event_name: The optional event_name string
3561  *
3562  * Given a variable hist_field, create a VAR_REF hist_field that
3563  * represents a reference to it.
3564  *
3565  * This function also adds the reference to the trigger that
3566  * now references the variable.
3567  *
3568  * Return: The VAR_REF field if successful, NULL if not
3569  */
3570 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
3571                                          struct hist_field *var_field,
3572                                          char *system, char *event_name)
3573 {
3574         unsigned long flags = HIST_FIELD_FL_VAR_REF;
3575         struct hist_field *ref_field;
3576         int i;
3577
3578         /* Check if the variable already exists */
3579         for (i = 0; i < hist_data->n_var_refs; i++) {
3580                 ref_field = hist_data->var_refs[i];
3581                 if (ref_field->var.idx == var_field->var.idx &&
3582                     ref_field->var.hist_data == var_field->hist_data) {
3583                         get_hist_field(ref_field);
3584                         return ref_field;
3585                 }
3586         }
3587
3588         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
3589         if (ref_field) {
3590                 if (init_var_ref(ref_field, var_field, system, event_name)) {
3591                         destroy_hist_field(ref_field, 0);
3592                         return NULL;
3593                 }
3594
3595                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3596                 ref_field->var_ref_idx = hist_data->n_var_refs++;
3597         }
3598
3599         return ref_field;
3600 }
3601
3602 static bool is_var_ref(char *var_name)
3603 {
3604         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
3605                 return false;
3606
3607         return true;
3608 }
3609
3610 static char *field_name_from_var(struct hist_trigger_data *hist_data,
3611                                  char *var_name)
3612 {
3613         char *name, *field;
3614         unsigned int i;
3615
3616         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3617                 name = hist_data->attrs->var_defs.name[i];
3618
3619                 if (strcmp(var_name, name) == 0) {
3620                         field = hist_data->attrs->var_defs.expr[i];
3621                         if (contains_operator(field) || is_var_ref(field))
3622                                 continue;
3623                         return field;
3624                 }
3625         }
3626
3627         return NULL;
3628 }
3629
3630 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
3631                                  char *system, char *event_name,
3632                                  char *var_name)
3633 {
3634         struct trace_event_call *call;
3635
3636         if (system && event_name) {
3637                 call = hist_data->event_file->event_call;
3638
3639                 if (strcmp(system, call->class->system) != 0)
3640                         return NULL;
3641
3642                 if (strcmp(event_name, trace_event_name(call)) != 0)
3643                         return NULL;
3644         }
3645
3646         if (!!system != !!event_name)
3647                 return NULL;
3648
3649         if (!is_var_ref(var_name))
3650                 return NULL;
3651
3652         var_name++;
3653
3654         return field_name_from_var(hist_data, var_name);
3655 }
3656
3657 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
3658                                         char *system, char *event_name,
3659                                         char *var_name)
3660 {
3661         struct hist_field *var_field = NULL, *ref_field = NULL;
3662         struct trace_array *tr = hist_data->event_file->tr;
3663
3664         if (!is_var_ref(var_name))
3665                 return NULL;
3666
3667         var_name++;
3668
3669         var_field = find_event_var(hist_data, system, event_name, var_name);
3670         if (var_field)
3671                 ref_field = create_var_ref(hist_data, var_field,
3672                                            system, event_name);
3673
3674         if (!ref_field)
3675                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
3676
3677         return ref_field;
3678 }
3679
3680 static struct ftrace_event_field *
3681 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
3682             char *field_str, unsigned long *flags)
3683 {
3684         struct ftrace_event_field *field = NULL;
3685         char *field_name, *modifier, *str;
3686         struct trace_array *tr = file->tr;
3687
3688         modifier = str = kstrdup(field_str, GFP_KERNEL);
3689         if (!modifier)
3690                 return ERR_PTR(-ENOMEM);
3691
3692         field_name = strsep(&modifier, ".");
3693         if (modifier) {
3694                 if (strcmp(modifier, "hex") == 0)
3695                         *flags |= HIST_FIELD_FL_HEX;
3696                 else if (strcmp(modifier, "sym") == 0)
3697                         *flags |= HIST_FIELD_FL_SYM;
3698                 else if (strcmp(modifier, "sym-offset") == 0)
3699                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
3700                 else if ((strcmp(modifier, "execname") == 0) &&
3701                          (strcmp(field_name, "common_pid") == 0))
3702                         *flags |= HIST_FIELD_FL_EXECNAME;
3703                 else if (strcmp(modifier, "syscall") == 0)
3704                         *flags |= HIST_FIELD_FL_SYSCALL;
3705                 else if (strcmp(modifier, "log2") == 0)
3706                         *flags |= HIST_FIELD_FL_LOG2;
3707                 else if (strcmp(modifier, "usecs") == 0)
3708                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
3709                 else {
3710                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
3711                         field = ERR_PTR(-EINVAL);
3712                         goto out;
3713                 }
3714         }
3715
3716         if (strcmp(field_name, "common_timestamp") == 0) {
3717                 *flags |= HIST_FIELD_FL_TIMESTAMP;
3718                 hist_data->enable_timestamps = true;
3719                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
3720                         hist_data->attrs->ts_in_usecs = true;
3721         } else if (strcmp(field_name, "cpu") == 0)
3722                 *flags |= HIST_FIELD_FL_CPU;
3723         else {
3724                 field = trace_find_event_field(file->event_call, field_name);
3725                 if (!field || !field->size) {
3726                         hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
3727                         field = ERR_PTR(-EINVAL);
3728                         goto out;
3729                 }
3730         }
3731  out:
3732         kfree(str);
3733
3734         return field;
3735 }
3736
3737 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
3738                                        struct hist_field *var_ref,
3739                                        char *var_name)
3740 {
3741         struct hist_field *alias = NULL;
3742         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
3743
3744         alias = create_hist_field(hist_data, NULL, flags, var_name);
3745         if (!alias)
3746                 return NULL;
3747
3748         alias->fn = var_ref->fn;
3749         alias->operands[0] = var_ref;
3750
3751         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
3752                 destroy_hist_field(alias, 0);
3753                 return NULL;
3754         }
3755
3756         alias->var_ref_idx = var_ref->var_ref_idx;
3757
3758         return alias;
3759 }
3760
3761 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
3762                                      struct trace_event_file *file, char *str,
3763                                      unsigned long *flags, char *var_name)
3764 {
3765         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
3766         struct ftrace_event_field *field = NULL;
3767         struct hist_field *hist_field = NULL;
3768         int ret = 0;
3769
3770         s = strchr(str, '.');
3771         if (s) {
3772                 s = strchr(++s, '.');
3773                 if (s) {
3774                         ref_system = strsep(&str, ".");
3775                         if (!str) {
3776                                 ret = -EINVAL;
3777                                 goto out;
3778                         }
3779                         ref_event = strsep(&str, ".");
3780                         if (!str) {
3781                                 ret = -EINVAL;
3782                                 goto out;
3783                         }
3784                         ref_var = str;
3785                 }
3786         }
3787
3788         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
3789         if (!s) {
3790                 hist_field = parse_var_ref(hist_data, ref_system,
3791                                            ref_event, ref_var);
3792                 if (hist_field) {
3793                         if (var_name) {
3794                                 hist_field = create_alias(hist_data, hist_field, var_name);
3795                                 if (!hist_field) {
3796                                         ret = -ENOMEM;
3797                                         goto out;
3798                                 }
3799                         }
3800                         return hist_field;
3801                 }
3802         } else
3803                 str = s;
3804
3805         field = parse_field(hist_data, file, str, flags);
3806         if (IS_ERR(field)) {
3807                 ret = PTR_ERR(field);
3808                 goto out;
3809         }
3810
3811         hist_field = create_hist_field(hist_data, field, *flags, var_name);
3812         if (!hist_field) {
3813                 ret = -ENOMEM;
3814                 goto out;
3815         }
3816
3817         return hist_field;
3818  out:
3819         return ERR_PTR(ret);
3820 }
3821
3822 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3823                                      struct trace_event_file *file,
3824                                      char *str, unsigned long flags,
3825                                      char *var_name, unsigned int level);
3826
3827 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
3828                                       struct trace_event_file *file,
3829                                       char *str, unsigned long flags,
3830                                       char *var_name, unsigned int level)
3831 {
3832         struct hist_field *operand1, *expr = NULL;
3833         unsigned long operand_flags;
3834         int ret = 0;
3835         char *s;
3836
3837         /* we support only -(xxx) i.e. explicit parens required */
3838
3839         if (level > 3) {
3840                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3841                 ret = -EINVAL;
3842                 goto free;
3843         }
3844
3845         str++; /* skip leading '-' */
3846
3847         s = strchr(str, '(');
3848         if (s)
3849                 str++;
3850         else {
3851                 ret = -EINVAL;
3852                 goto free;
3853         }
3854
3855         s = strrchr(str, ')');
3856         if (s)
3857                 *s = '\0';
3858         else {
3859                 ret = -EINVAL; /* no closing ')' */
3860                 goto free;
3861         }
3862
3863         flags |= HIST_FIELD_FL_EXPR;
3864         expr = create_hist_field(hist_data, NULL, flags, var_name);
3865         if (!expr) {
3866                 ret = -ENOMEM;
3867                 goto free;
3868         }
3869
3870         operand_flags = 0;
3871         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3872         if (IS_ERR(operand1)) {
3873                 ret = PTR_ERR(operand1);
3874                 goto free;
3875         }
3876
3877         expr->flags |= operand1->flags &
3878                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3879         expr->fn = hist_field_unary_minus;
3880         expr->operands[0] = operand1;
3881         expr->operator = FIELD_OP_UNARY_MINUS;
3882         expr->name = expr_str(expr, 0);
3883         expr->type = kstrdup(operand1->type, GFP_KERNEL);
3884         if (!expr->type) {
3885                 ret = -ENOMEM;
3886                 goto free;
3887         }
3888
3889         return expr;
3890  free:
3891         destroy_hist_field(expr, 0);
3892         return ERR_PTR(ret);
3893 }
3894
3895 static int check_expr_operands(struct trace_array *tr,
3896                                struct hist_field *operand1,
3897                                struct hist_field *operand2)
3898 {
3899         unsigned long operand1_flags = operand1->flags;
3900         unsigned long operand2_flags = operand2->flags;
3901
3902         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
3903             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
3904                 struct hist_field *var;
3905
3906                 var = find_var_field(operand1->var.hist_data, operand1->name);
3907                 if (!var)
3908                         return -EINVAL;
3909                 operand1_flags = var->flags;
3910         }
3911
3912         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
3913             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
3914                 struct hist_field *var;
3915
3916                 var = find_var_field(operand2->var.hist_data, operand2->name);
3917                 if (!var)
3918                         return -EINVAL;
3919                 operand2_flags = var->flags;
3920         }
3921
3922         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
3923             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
3924                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
3925                 return -EINVAL;
3926         }
3927
3928         return 0;
3929 }
3930
3931 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
3932                                      struct trace_event_file *file,
3933                                      char *str, unsigned long flags,
3934                                      char *var_name, unsigned int level)
3935 {
3936         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
3937         unsigned long operand_flags;
3938         int field_op, ret = -EINVAL;
3939         char *sep, *operand1_str;
3940
3941         if (level > 3) {
3942                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
3943                 return ERR_PTR(-EINVAL);
3944         }
3945
3946         field_op = contains_operator(str);
3947
3948         if (field_op == FIELD_OP_NONE)
3949                 return parse_atom(hist_data, file, str, &flags, var_name);
3950
3951         if (field_op == FIELD_OP_UNARY_MINUS)
3952                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
3953
3954         switch (field_op) {
3955         case FIELD_OP_MINUS:
3956                 sep = "-";
3957                 break;
3958         case FIELD_OP_PLUS:
3959                 sep = "+";
3960                 break;
3961         default:
3962                 goto free;
3963         }
3964
3965         operand1_str = strsep(&str, sep);
3966         if (!operand1_str || !str)
3967                 goto free;
3968
3969         operand_flags = 0;
3970         operand1 = parse_atom(hist_data, file, operand1_str,
3971                               &operand_flags, NULL);
3972         if (IS_ERR(operand1)) {
3973                 ret = PTR_ERR(operand1);
3974                 operand1 = NULL;
3975                 goto free;
3976         }
3977
3978         /* rest of string could be another expression e.g. b+c in a+b+c */
3979         operand_flags = 0;
3980         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
3981         if (IS_ERR(operand2)) {
3982                 ret = PTR_ERR(operand2);
3983                 operand2 = NULL;
3984                 goto free;
3985         }
3986
3987         ret = check_expr_operands(file->tr, operand1, operand2);
3988         if (ret)
3989                 goto free;
3990
3991         flags |= HIST_FIELD_FL_EXPR;
3992
3993         flags |= operand1->flags &
3994                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
3995
3996         expr = create_hist_field(hist_data, NULL, flags, var_name);
3997         if (!expr) {
3998                 ret = -ENOMEM;
3999                 goto free;
4000         }
4001
4002         operand1->read_once = true;
4003         operand2->read_once = true;
4004
4005         expr->operands[0] = operand1;
4006         expr->operands[1] = operand2;
4007         expr->operator = field_op;
4008         expr->name = expr_str(expr, 0);
4009         expr->type = kstrdup(operand1->type, GFP_KERNEL);
4010         if (!expr->type) {
4011                 ret = -ENOMEM;
4012                 goto free;
4013         }
4014
4015         switch (field_op) {
4016         case FIELD_OP_MINUS:
4017                 expr->fn = hist_field_minus;
4018                 break;
4019         case FIELD_OP_PLUS:
4020                 expr->fn = hist_field_plus;
4021                 break;
4022         default:
4023                 ret = -EINVAL;
4024                 goto free;
4025         }
4026
4027         return expr;
4028  free:
4029         destroy_hist_field(operand1, 0);
4030         destroy_hist_field(operand2, 0);
4031         destroy_hist_field(expr, 0);
4032
4033         return ERR_PTR(ret);
4034 }
4035
4036 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
4037                                  struct trace_event_file *file)
4038 {
4039         struct event_trigger_data *test;
4040
4041         lockdep_assert_held(&event_mutex);
4042
4043         list_for_each_entry(test, &file->triggers, list) {
4044                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
4045                         if (test->private_data == hist_data)
4046                                 return test->filter_str;
4047                 }
4048         }
4049
4050         return NULL;
4051 }
4052
4053 static struct event_command trigger_hist_cmd;
4054 static int event_hist_trigger_func(struct event_command *cmd_ops,
4055                                    struct trace_event_file *file,
4056                                    char *glob, char *cmd, char *param);
4057
4058 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
4059                             struct hist_trigger_data *hist_data,
4060                             unsigned int n_keys)
4061 {
4062         struct hist_field *target_hist_field, *hist_field;
4063         unsigned int n, i, j;
4064
4065         if (hist_data->n_fields - hist_data->n_vals != n_keys)
4066                 return false;
4067
4068         i = hist_data->n_vals;
4069         j = target_hist_data->n_vals;
4070
4071         for (n = 0; n < n_keys; n++) {
4072                 hist_field = hist_data->fields[i + n];
4073                 target_hist_field = target_hist_data->fields[j + n];
4074
4075                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
4076                         return false;
4077                 if (hist_field->size != target_hist_field->size)
4078                         return false;
4079                 if (hist_field->is_signed != target_hist_field->is_signed)
4080                         return false;
4081         }
4082
4083         return true;
4084 }
4085
4086 static struct hist_trigger_data *
4087 find_compatible_hist(struct hist_trigger_data *target_hist_data,
4088                      struct trace_event_file *file)
4089 {
4090         struct hist_trigger_data *hist_data;
4091         struct event_trigger_data *test;
4092         unsigned int n_keys;
4093
4094         lockdep_assert_held(&event_mutex);
4095
4096         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
4097
4098         list_for_each_entry(test, &file->triggers, list) {
4099                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
4100                         hist_data = test->private_data;
4101
4102                         if (compatible_keys(target_hist_data, hist_data, n_keys))
4103                                 return hist_data;
4104                 }
4105         }
4106
4107         return NULL;
4108 }
4109
4110 static struct trace_event_file *event_file(struct trace_array *tr,
4111                                            char *system, char *event_name)
4112 {
4113         struct trace_event_file *file;
4114
4115         file = __find_event_file(tr, system, event_name);
4116         if (!file)
4117                 return ERR_PTR(-EINVAL);
4118
4119         return file;
4120 }
4121
4122 static struct hist_field *
4123 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
4124                          char *system, char *event_name, char *field_name)
4125 {
4126         struct hist_field *event_var;
4127         char *synthetic_name;
4128
4129         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
4130         if (!synthetic_name)
4131                 return ERR_PTR(-ENOMEM);
4132
4133         strcpy(synthetic_name, "synthetic_");
4134         strcat(synthetic_name, field_name);
4135
4136         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
4137
4138         kfree(synthetic_name);
4139
4140         return event_var;
4141 }
4142
4143 /**
4144  * create_field_var_hist - Automatically create a histogram and var for a field
4145  * @target_hist_data: The target hist trigger
4146  * @subsys_name: Optional subsystem name
4147  * @event_name: Optional event name
4148  * @field_name: The name of the field (and the resulting variable)
4149  *
4150  * Hist trigger actions fetch data from variables, not directly from
4151  * events.  However, for convenience, users are allowed to directly
4152  * specify an event field in an action, which will be automatically
4153  * converted into a variable on their behalf.
4154
4155  * If a user specifies a field on an event that isn't the event the
4156  * histogram currently being defined (the target event histogram), the
4157  * only way that can be accomplished is if a new hist trigger is
4158  * created and the field variable defined on that.
4159  *
4160  * This function creates a new histogram compatible with the target
4161  * event (meaning a histogram with the same key as the target
4162  * histogram), and creates a variable for the specified field, but
4163  * with 'synthetic_' prepended to the variable name in order to avoid
4164  * collision with normal field variables.
4165  *
4166  * Return: The variable created for the field.
4167  */
4168 static struct hist_field *
4169 create_field_var_hist(struct hist_trigger_data *target_hist_data,
4170                       char *subsys_name, char *event_name, char *field_name)
4171 {
4172         struct trace_array *tr = target_hist_data->event_file->tr;
4173         struct hist_field *event_var = ERR_PTR(-EINVAL);
4174         struct hist_trigger_data *hist_data;
4175         unsigned int i, n, first = true;
4176         struct field_var_hist *var_hist;
4177         struct trace_event_file *file;
4178         struct hist_field *key_field;
4179         char *saved_filter;
4180         char *cmd;
4181         int ret;
4182
4183         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
4184                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
4185                 return ERR_PTR(-EINVAL);
4186         }
4187
4188         file = event_file(tr, subsys_name, event_name);
4189
4190         if (IS_ERR(file)) {
4191                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
4192                 ret = PTR_ERR(file);
4193                 return ERR_PTR(ret);
4194         }
4195
4196         /*
4197          * Look for a histogram compatible with target.  We'll use the
4198          * found histogram specification to create a new matching
4199          * histogram with our variable on it.  target_hist_data is not
4200          * yet a registered histogram so we can't use that.
4201          */
4202         hist_data = find_compatible_hist(target_hist_data, file);
4203         if (!hist_data) {
4204                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
4205                 return ERR_PTR(-EINVAL);
4206         }
4207
4208         /* See if a synthetic field variable has already been created */
4209         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
4210                                              event_name, field_name);
4211         if (!IS_ERR_OR_NULL(event_var))
4212                 return event_var;
4213
4214         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
4215         if (!var_hist)
4216                 return ERR_PTR(-ENOMEM);
4217
4218         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
4219         if (!cmd) {
4220                 kfree(var_hist);
4221                 return ERR_PTR(-ENOMEM);
4222         }
4223
4224         /* Use the same keys as the compatible histogram */
4225         strcat(cmd, "keys=");
4226
4227         for_each_hist_key_field(i, hist_data) {
4228                 key_field = hist_data->fields[i];
4229                 if (!first)
4230                         strcat(cmd, ",");
4231                 strcat(cmd, key_field->field->name);
4232                 first = false;
4233         }
4234
4235         /* Create the synthetic field variable specification */
4236         strcat(cmd, ":synthetic_");
4237         strcat(cmd, field_name);
4238         strcat(cmd, "=");
4239         strcat(cmd, field_name);
4240
4241         /* Use the same filter as the compatible histogram */
4242         saved_filter = find_trigger_filter(hist_data, file);
4243         if (saved_filter) {
4244                 strcat(cmd, " if ");
4245                 strcat(cmd, saved_filter);
4246         }
4247
4248         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
4249         if (!var_hist->cmd) {
4250                 kfree(cmd);
4251                 kfree(var_hist);
4252                 return ERR_PTR(-ENOMEM);
4253         }
4254
4255         /* Save the compatible histogram information */
4256         var_hist->hist_data = hist_data;
4257
4258         /* Create the new histogram with our variable */
4259         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
4260                                       "", "hist", cmd);
4261         if (ret) {
4262                 kfree(cmd);
4263                 kfree(var_hist->cmd);
4264                 kfree(var_hist);
4265                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
4266                 return ERR_PTR(ret);
4267         }
4268
4269         kfree(cmd);
4270
4271         /* If we can't find the variable, something went wrong */
4272         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
4273                                              event_name, field_name);
4274         if (IS_ERR_OR_NULL(event_var)) {
4275                 kfree(var_hist->cmd);
4276                 kfree(var_hist);
4277                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
4278                 return ERR_PTR(-EINVAL);
4279         }
4280
4281         n = target_hist_data->n_field_var_hists;
4282         target_hist_data->field_var_hists[n] = var_hist;
4283         target_hist_data->n_field_var_hists++;
4284
4285         return event_var;
4286 }
4287
4288 static struct hist_field *
4289 find_target_event_var(struct hist_trigger_data *hist_data,
4290                       char *subsys_name, char *event_name, char *var_name)
4291 {
4292         struct trace_event_file *file = hist_data->event_file;
4293         struct hist_field *hist_field = NULL;
4294
4295         if (subsys_name) {
4296                 struct trace_event_call *call;
4297
4298                 if (!event_name)
4299                         return NULL;
4300
4301                 call = file->event_call;
4302
4303                 if (strcmp(subsys_name, call->class->system) != 0)
4304                         return NULL;
4305
4306                 if (strcmp(event_name, trace_event_name(call)) != 0)
4307                         return NULL;
4308         }
4309
4310         hist_field = find_var_field(hist_data, var_name);
4311
4312         return hist_field;
4313 }
4314
4315 static inline void __update_field_vars(struct tracing_map_elt *elt,
4316                                        struct ring_buffer_event *rbe,
4317                                        void *rec,
4318                                        struct field_var **field_vars,
4319                                        unsigned int n_field_vars,
4320                                        unsigned int field_var_str_start)
4321 {
4322         struct hist_elt_data *elt_data = elt->private_data;
4323         unsigned int i, j, var_idx;
4324         u64 var_val;
4325
4326         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
4327                 struct field_var *field_var = field_vars[i];
4328                 struct hist_field *var = field_var->var;
4329                 struct hist_field *val = field_var->val;
4330
4331                 var_val = val->fn(val, elt, rbe, rec);
4332                 var_idx = var->var.idx;
4333
4334                 if (val->flags & HIST_FIELD_FL_STRING) {
4335                         char *str = elt_data->field_var_str[j++];
4336                         char *val_str = (char *)(uintptr_t)var_val;
4337
4338                         strscpy(str, val_str, STR_VAR_LEN_MAX);
4339                         var_val = (u64)(uintptr_t)str;
4340                 }
4341                 tracing_map_set_var(elt, var_idx, var_val);
4342         }
4343 }
4344
4345 static void update_field_vars(struct hist_trigger_data *hist_data,
4346                               struct tracing_map_elt *elt,
4347                               struct ring_buffer_event *rbe,
4348                               void *rec)
4349 {
4350         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
4351                             hist_data->n_field_vars, 0);
4352 }
4353
4354 static void save_track_data_vars(struct hist_trigger_data *hist_data,
4355                                  struct tracing_map_elt *elt, void *rec,
4356                                  struct ring_buffer_event *rbe, void *key,
4357                                  struct action_data *data, u64 *var_ref_vals)
4358 {
4359         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
4360                             hist_data->n_save_vars, hist_data->n_field_var_str);
4361 }
4362
4363 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
4364                                      struct trace_event_file *file,
4365                                      char *name, int size, const char *type)
4366 {
4367         struct hist_field *var;
4368         int idx;
4369
4370         if (find_var(hist_data, file, name) && !hist_data->remove) {
4371                 var = ERR_PTR(-EINVAL);
4372                 goto out;
4373         }
4374
4375         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
4376         if (!var) {
4377                 var = ERR_PTR(-ENOMEM);
4378                 goto out;
4379         }
4380
4381         idx = tracing_map_add_var(hist_data->map);
4382         if (idx < 0) {
4383                 kfree(var);
4384                 var = ERR_PTR(-EINVAL);
4385                 goto out;
4386         }
4387
4388         var->ref = 1;
4389         var->flags = HIST_FIELD_FL_VAR;
4390         var->var.idx = idx;
4391         var->var.hist_data = var->hist_data = hist_data;
4392         var->size = size;
4393         var->var.name = kstrdup(name, GFP_KERNEL);
4394         var->type = kstrdup(type, GFP_KERNEL);
4395         if (!var->var.name || !var->type) {
4396                 kfree(var->var.name);
4397                 kfree(var->type);
4398                 kfree(var);
4399                 var = ERR_PTR(-ENOMEM);
4400         }
4401  out:
4402         return var;
4403 }
4404
4405 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
4406                                           struct trace_event_file *file,
4407                                           char *field_name)
4408 {
4409         struct hist_field *val = NULL, *var = NULL;
4410         unsigned long flags = HIST_FIELD_FL_VAR;
4411         struct trace_array *tr = file->tr;
4412         struct field_var *field_var;
4413         int ret = 0;
4414
4415         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
4416                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
4417                 ret = -EINVAL;
4418                 goto err;
4419         }
4420
4421         val = parse_atom(hist_data, file, field_name, &flags, NULL);
4422         if (IS_ERR(val)) {
4423                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
4424                 ret = PTR_ERR(val);
4425                 goto err;
4426         }
4427
4428         var = create_var(hist_data, file, field_name, val->size, val->type);
4429         if (IS_ERR(var)) {
4430                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
4431                 kfree(val);
4432                 ret = PTR_ERR(var);
4433                 goto err;
4434         }
4435
4436         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
4437         if (!field_var) {
4438                 kfree(val);
4439                 kfree(var);
4440                 ret =  -ENOMEM;
4441                 goto err;
4442         }
4443
4444         field_var->var = var;
4445         field_var->val = val;
4446  out:
4447         return field_var;
4448  err:
4449         field_var = ERR_PTR(ret);
4450         goto out;
4451 }
4452
4453 /**
4454  * create_target_field_var - Automatically create a variable for a field
4455  * @target_hist_data: The target hist trigger
4456  * @subsys_name: Optional subsystem name
4457  * @event_name: Optional event name
4458  * @var_name: The name of the field (and the resulting variable)
4459  *
4460  * Hist trigger actions fetch data from variables, not directly from
4461  * events.  However, for convenience, users are allowed to directly
4462  * specify an event field in an action, which will be automatically
4463  * converted into a variable on their behalf.
4464
4465  * This function creates a field variable with the name var_name on
4466  * the hist trigger currently being defined on the target event.  If
4467  * subsys_name and event_name are specified, this function simply
4468  * verifies that they do in fact match the target event subsystem and
4469  * event name.
4470  *
4471  * Return: The variable created for the field.
4472  */
4473 static struct field_var *
4474 create_target_field_var(struct hist_trigger_data *target_hist_data,
4475                         char *subsys_name, char *event_name, char *var_name)
4476 {
4477         struct trace_event_file *file = target_hist_data->event_file;
4478
4479         if (subsys_name) {
4480                 struct trace_event_call *call;
4481
4482                 if (!event_name)
4483                         return NULL;
4484
4485                 call = file->event_call;
4486
4487                 if (strcmp(subsys_name, call->class->system) != 0)
4488                         return NULL;
4489
4490                 if (strcmp(event_name, trace_event_name(call)) != 0)
4491                         return NULL;
4492         }
4493
4494         return create_field_var(target_hist_data, file, var_name);
4495 }
4496
4497 static bool check_track_val_max(u64 track_val, u64 var_val)
4498 {
4499         if (var_val <= track_val)
4500                 return false;
4501
4502         return true;
4503 }
4504
4505 static bool check_track_val_changed(u64 track_val, u64 var_val)
4506 {
4507         if (var_val == track_val)
4508                 return false;
4509
4510         return true;
4511 }
4512
4513 static u64 get_track_val(struct hist_trigger_data *hist_data,
4514                          struct tracing_map_elt *elt,
4515                          struct action_data *data)
4516 {
4517         unsigned int track_var_idx = data->track_data.track_var->var.idx;
4518         u64 track_val;
4519
4520         track_val = tracing_map_read_var(elt, track_var_idx);
4521
4522         return track_val;
4523 }
4524
4525 static void save_track_val(struct hist_trigger_data *hist_data,
4526                            struct tracing_map_elt *elt,
4527                            struct action_data *data, u64 var_val)
4528 {
4529         unsigned int track_var_idx = data->track_data.track_var->var.idx;
4530
4531         tracing_map_set_var(elt, track_var_idx, var_val);
4532 }
4533
4534 static void save_track_data(struct hist_trigger_data *hist_data,
4535                             struct tracing_map_elt *elt, void *rec,
4536                             struct ring_buffer_event *rbe, void *key,
4537                             struct action_data *data, u64 *var_ref_vals)
4538 {
4539         if (data->track_data.save_data)
4540                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4541 }
4542
4543 static bool check_track_val(struct tracing_map_elt *elt,
4544                             struct action_data *data,
4545                             u64 var_val)
4546 {
4547         struct hist_trigger_data *hist_data;
4548         u64 track_val;
4549
4550         hist_data = data->track_data.track_var->hist_data;
4551         track_val = get_track_val(hist_data, elt, data);
4552
4553         return data->track_data.check_val(track_val, var_val);
4554 }
4555
4556 #ifdef CONFIG_TRACER_SNAPSHOT
4557 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
4558 {
4559         /* called with tr->max_lock held */
4560         struct track_data *track_data = tr->cond_snapshot->cond_data;
4561         struct hist_elt_data *elt_data, *track_elt_data;
4562         struct snapshot_context *context = cond_data;
4563         struct action_data *action;
4564         u64 track_val;
4565
4566         if (!track_data)
4567                 return false;
4568
4569         action = track_data->action_data;
4570
4571         track_val = get_track_val(track_data->hist_data, context->elt,
4572                                   track_data->action_data);
4573
4574         if (!action->track_data.check_val(track_data->track_val, track_val))
4575                 return false;
4576
4577         track_data->track_val = track_val;
4578         memcpy(track_data->key, context->key, track_data->key_len);
4579
4580         elt_data = context->elt->private_data;
4581         track_elt_data = track_data->elt.private_data;
4582         if (elt_data->comm)
4583                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
4584
4585         track_data->updated = true;
4586
4587         return true;
4588 }
4589
4590 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
4591                                      struct tracing_map_elt *elt, void *rec,
4592                                      struct ring_buffer_event *rbe, void *key,
4593                                      struct action_data *data,
4594                                      u64 *var_ref_vals)
4595 {
4596         struct trace_event_file *file = hist_data->event_file;
4597         struct snapshot_context context;
4598
4599         context.elt = elt;
4600         context.key = key;
4601
4602         tracing_snapshot_cond(file->tr, &context);
4603 }
4604
4605 static void hist_trigger_print_key(struct seq_file *m,
4606                                    struct hist_trigger_data *hist_data,
4607                                    void *key,
4608                                    struct tracing_map_elt *elt);
4609
4610 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
4611 {
4612         unsigned int i;
4613
4614         if (!hist_data->n_actions)
4615                 return NULL;
4616
4617         for (i = 0; i < hist_data->n_actions; i++) {
4618                 struct action_data *data = hist_data->actions[i];
4619
4620                 if (data->action == ACTION_SNAPSHOT)
4621                         return data;
4622         }
4623
4624         return NULL;
4625 }
4626
4627 static void track_data_snapshot_print(struct seq_file *m,
4628                                       struct hist_trigger_data *hist_data)
4629 {
4630         struct trace_event_file *file = hist_data->event_file;
4631         struct track_data *track_data;
4632         struct action_data *action;
4633
4634         track_data = tracing_cond_snapshot_data(file->tr);
4635         if (!track_data)
4636                 return;
4637
4638         if (!track_data->updated)
4639                 return;
4640
4641         action = snapshot_action(hist_data);
4642         if (!action)
4643                 return;
4644
4645         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
4646         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
4647                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
4648                    action->track_data.var_str, track_data->track_val);
4649
4650         seq_puts(m, "\ttriggered by event with key: ");
4651         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
4652         seq_putc(m, '\n');
4653 }
4654 #else
4655 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
4656 {
4657         return false;
4658 }
4659 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
4660                                      struct tracing_map_elt *elt, void *rec,
4661                                      struct ring_buffer_event *rbe, void *key,
4662                                      struct action_data *data,
4663                                      u64 *var_ref_vals) {}
4664 static void track_data_snapshot_print(struct seq_file *m,
4665                                       struct hist_trigger_data *hist_data) {}
4666 #endif /* CONFIG_TRACER_SNAPSHOT */
4667
4668 static void track_data_print(struct seq_file *m,
4669                              struct hist_trigger_data *hist_data,
4670                              struct tracing_map_elt *elt,
4671                              struct action_data *data)
4672 {
4673         u64 track_val = get_track_val(hist_data, elt, data);
4674         unsigned int i, save_var_idx;
4675
4676         if (data->handler == HANDLER_ONMAX)
4677                 seq_printf(m, "\n\tmax: %10llu", track_val);
4678         else if (data->handler == HANDLER_ONCHANGE)
4679                 seq_printf(m, "\n\tchanged: %10llu", track_val);
4680
4681         if (data->action == ACTION_SNAPSHOT)
4682                 return;
4683
4684         for (i = 0; i < hist_data->n_save_vars; i++) {
4685                 struct hist_field *save_val = hist_data->save_vars[i]->val;
4686                 struct hist_field *save_var = hist_data->save_vars[i]->var;
4687                 u64 val;
4688
4689                 save_var_idx = save_var->var.idx;
4690
4691                 val = tracing_map_read_var(elt, save_var_idx);
4692
4693                 if (save_val->flags & HIST_FIELD_FL_STRING) {
4694                         seq_printf(m, "  %s: %-32s", save_var->var.name,
4695                                    (char *)(uintptr_t)(val));
4696                 } else
4697                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
4698         }
4699 }
4700
4701 static void ontrack_action(struct hist_trigger_data *hist_data,
4702                            struct tracing_map_elt *elt, void *rec,
4703                            struct ring_buffer_event *rbe, void *key,
4704                            struct action_data *data, u64 *var_ref_vals)
4705 {
4706         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
4707
4708         if (check_track_val(elt, data, var_val)) {
4709                 save_track_val(hist_data, elt, data, var_val);
4710                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4711         }
4712 }
4713
4714 static void action_data_destroy(struct action_data *data)
4715 {
4716         unsigned int i;
4717
4718         lockdep_assert_held(&event_mutex);
4719
4720         kfree(data->action_name);
4721
4722         for (i = 0; i < data->n_params; i++)
4723                 kfree(data->params[i]);
4724
4725         if (data->synth_event)
4726                 data->synth_event->ref--;
4727
4728         kfree(data->synth_event_name);
4729
4730         kfree(data);
4731 }
4732
4733 static void track_data_destroy(struct hist_trigger_data *hist_data,
4734                                struct action_data *data)
4735 {
4736         struct trace_event_file *file = hist_data->event_file;
4737
4738         destroy_hist_field(data->track_data.track_var, 0);
4739
4740         if (data->action == ACTION_SNAPSHOT) {
4741                 struct track_data *track_data;
4742
4743                 track_data = tracing_cond_snapshot_data(file->tr);
4744                 if (track_data && track_data->hist_data == hist_data) {
4745                         tracing_snapshot_cond_disable(file->tr);
4746                         track_data_free(track_data);
4747                 }
4748         }
4749
4750         kfree(data->track_data.var_str);
4751
4752         action_data_destroy(data);
4753 }
4754
4755 static int action_create(struct hist_trigger_data *hist_data,
4756                          struct action_data *data);
4757
4758 static int track_data_create(struct hist_trigger_data *hist_data,
4759                              struct action_data *data)
4760 {
4761         struct hist_field *var_field, *ref_field, *track_var = NULL;
4762         struct trace_event_file *file = hist_data->event_file;
4763         struct trace_array *tr = file->tr;
4764         char *track_data_var_str;
4765         int ret = 0;
4766
4767         track_data_var_str = data->track_data.var_str;
4768         if (track_data_var_str[0] != '$') {
4769                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
4770                 return -EINVAL;
4771         }
4772         track_data_var_str++;
4773
4774         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
4775         if (!var_field) {
4776                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
4777                 return -EINVAL;
4778         }
4779
4780         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
4781         if (!ref_field)
4782                 return -ENOMEM;
4783
4784         data->track_data.var_ref = ref_field;
4785
4786         if (data->handler == HANDLER_ONMAX)
4787                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
4788         if (IS_ERR(track_var)) {
4789                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
4790                 ret = PTR_ERR(track_var);
4791                 goto out;
4792         }
4793
4794         if (data->handler == HANDLER_ONCHANGE)
4795                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
4796         if (IS_ERR(track_var)) {
4797                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
4798                 ret = PTR_ERR(track_var);
4799                 goto out;
4800         }
4801         data->track_data.track_var = track_var;
4802
4803         ret = action_create(hist_data, data);
4804  out:
4805         return ret;
4806 }
4807
4808 static int parse_action_params(struct trace_array *tr, char *params,
4809                                struct action_data *data)
4810 {
4811         char *param, *saved_param;
4812         bool first_param = true;
4813         int ret = 0;
4814
4815         while (params) {
4816                 if (data->n_params >= SYNTH_FIELDS_MAX) {
4817                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
4818                         goto out;
4819                 }
4820
4821                 param = strsep(&params, ",");
4822                 if (!param) {
4823                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
4824                         ret = -EINVAL;
4825                         goto out;
4826                 }
4827
4828                 param = strstrip(param);
4829                 if (strlen(param) < 2) {
4830                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
4831                         ret = -EINVAL;
4832                         goto out;
4833                 }
4834
4835                 saved_param = kstrdup(param, GFP_KERNEL);
4836                 if (!saved_param) {
4837                         ret = -ENOMEM;
4838                         goto out;
4839                 }
4840
4841                 if (first_param && data->use_trace_keyword) {
4842                         data->synth_event_name = saved_param;
4843                         first_param = false;
4844                         continue;
4845                 }
4846                 first_param = false;
4847
4848                 data->params[data->n_params++] = saved_param;
4849         }
4850  out:
4851         return ret;
4852 }
4853
4854 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
4855                         enum handler_id handler)
4856 {
4857         char *action_name;
4858         int ret = 0;
4859
4860         strsep(&str, ".");
4861         if (!str) {
4862                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
4863                 ret = -EINVAL;
4864                 goto out;
4865         }
4866
4867         action_name = strsep(&str, "(");
4868         if (!action_name || !str) {
4869                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
4870                 ret = -EINVAL;
4871                 goto out;
4872         }
4873
4874         if (str_has_prefix(action_name, "save")) {
4875                 char *params = strsep(&str, ")");
4876
4877                 if (!params) {
4878                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
4879                         ret = -EINVAL;
4880                         goto out;
4881                 }
4882
4883                 ret = parse_action_params(tr, params, data);
4884                 if (ret)
4885                         goto out;
4886
4887                 if (handler == HANDLER_ONMAX)
4888                         data->track_data.check_val = check_track_val_max;
4889                 else if (handler == HANDLER_ONCHANGE)
4890                         data->track_data.check_val = check_track_val_changed;
4891                 else {
4892                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4893                         ret = -EINVAL;
4894                         goto out;
4895                 }
4896
4897                 data->track_data.save_data = save_track_data_vars;
4898                 data->fn = ontrack_action;
4899                 data->action = ACTION_SAVE;
4900         } else if (str_has_prefix(action_name, "snapshot")) {
4901                 char *params = strsep(&str, ")");
4902
4903                 if (!str) {
4904                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
4905                         ret = -EINVAL;
4906                         goto out;
4907                 }
4908
4909                 if (handler == HANDLER_ONMAX)
4910                         data->track_data.check_val = check_track_val_max;
4911                 else if (handler == HANDLER_ONCHANGE)
4912                         data->track_data.check_val = check_track_val_changed;
4913                 else {
4914                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
4915                         ret = -EINVAL;
4916                         goto out;
4917                 }
4918
4919                 data->track_data.save_data = save_track_data_snapshot;
4920                 data->fn = ontrack_action;
4921                 data->action = ACTION_SNAPSHOT;
4922         } else {
4923                 char *params = strsep(&str, ")");
4924
4925                 if (str_has_prefix(action_name, "trace"))
4926                         data->use_trace_keyword = true;
4927
4928                 if (params) {
4929                         ret = parse_action_params(tr, params, data);
4930                         if (ret)
4931                                 goto out;
4932                 }
4933
4934                 if (handler == HANDLER_ONMAX)
4935                         data->track_data.check_val = check_track_val_max;
4936                 else if (handler == HANDLER_ONCHANGE)
4937                         data->track_data.check_val = check_track_val_changed;
4938
4939                 if (handler != HANDLER_ONMATCH) {
4940                         data->track_data.save_data = action_trace;
4941                         data->fn = ontrack_action;
4942                 } else
4943                         data->fn = action_trace;
4944
4945                 data->action = ACTION_TRACE;
4946         }
4947
4948         data->action_name = kstrdup(action_name, GFP_KERNEL);
4949         if (!data->action_name) {
4950                 ret = -ENOMEM;
4951                 goto out;
4952         }
4953
4954         data->handler = handler;
4955  out:
4956         return ret;
4957 }
4958
4959 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
4960                                             char *str, enum handler_id handler)
4961 {
4962         struct action_data *data;
4963         int ret = -EINVAL;
4964         char *var_str;
4965
4966         data = kzalloc(sizeof(*data), GFP_KERNEL);
4967         if (!data)
4968                 return ERR_PTR(-ENOMEM);
4969
4970         var_str = strsep(&str, ")");
4971         if (!var_str || !str) {
4972                 ret = -EINVAL;
4973                 goto free;
4974         }
4975
4976         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
4977         if (!data->track_data.var_str) {
4978                 ret = -ENOMEM;
4979                 goto free;
4980         }
4981
4982         ret = action_parse(hist_data->event_file->tr, str, data, handler);
4983         if (ret)
4984                 goto free;
4985  out:
4986         return data;
4987  free:
4988         track_data_destroy(hist_data, data);
4989         data = ERR_PTR(ret);
4990         goto out;
4991 }
4992
4993 static void onmatch_destroy(struct action_data *data)
4994 {
4995         kfree(data->match_data.event);
4996         kfree(data->match_data.event_system);
4997
4998         action_data_destroy(data);
4999 }
5000
5001 static void destroy_field_var(struct field_var *field_var)
5002 {
5003         if (!field_var)
5004                 return;
5005
5006         destroy_hist_field(field_var->var, 0);
5007         destroy_hist_field(field_var->val, 0);
5008
5009         kfree(field_var);
5010 }
5011
5012 static void destroy_field_vars(struct hist_trigger_data *hist_data)
5013 {
5014         unsigned int i;
5015
5016         for (i = 0; i < hist_data->n_field_vars; i++)
5017                 destroy_field_var(hist_data->field_vars[i]);
5018
5019         for (i = 0; i < hist_data->n_save_vars; i++)
5020                 destroy_field_var(hist_data->save_vars[i]);
5021 }
5022
5023 static void save_field_var(struct hist_trigger_data *hist_data,
5024                            struct field_var *field_var)
5025 {
5026         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
5027
5028         if (field_var->val->flags & HIST_FIELD_FL_STRING)
5029                 hist_data->n_field_var_str++;
5030 }
5031
5032
5033 static int check_synth_field(struct synth_event *event,
5034                              struct hist_field *hist_field,
5035                              unsigned int field_pos)
5036 {
5037         struct synth_field *field;
5038
5039         if (field_pos >= event->n_fields)
5040                 return -EINVAL;
5041
5042         field = event->fields[field_pos];
5043
5044         if (strcmp(field->type, hist_field->type) != 0) {
5045                 if (field->size != hist_field->size ||
5046                     field->is_signed != hist_field->is_signed)
5047                         return -EINVAL;
5048         }
5049
5050         return 0;
5051 }
5052
5053 static struct hist_field *
5054 trace_action_find_var(struct hist_trigger_data *hist_data,
5055                       struct action_data *data,
5056                       char *system, char *event, char *var)
5057 {
5058         struct trace_array *tr = hist_data->event_file->tr;
5059         struct hist_field *hist_field;
5060
5061         var++; /* skip '$' */
5062
5063         hist_field = find_target_event_var(hist_data, system, event, var);
5064         if (!hist_field) {
5065                 if (!system && data->handler == HANDLER_ONMATCH) {
5066                         system = data->match_data.event_system;
5067                         event = data->match_data.event;
5068                 }
5069
5070                 hist_field = find_event_var(hist_data, system, event, var);
5071         }
5072
5073         if (!hist_field)
5074                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
5075
5076         return hist_field;
5077 }
5078
5079 static struct hist_field *
5080 trace_action_create_field_var(struct hist_trigger_data *hist_data,
5081                               struct action_data *data, char *system,
5082                               char *event, char *var)
5083 {
5084         struct hist_field *hist_field = NULL;
5085         struct field_var *field_var;
5086
5087         /*
5088          * First try to create a field var on the target event (the
5089          * currently being defined).  This will create a variable for
5090          * unqualified fields on the target event, or if qualified,
5091          * target fields that have qualified names matching the target.
5092          */
5093         field_var = create_target_field_var(hist_data, system, event, var);
5094
5095         if (field_var && !IS_ERR(field_var)) {
5096                 save_field_var(hist_data, field_var);
5097                 hist_field = field_var->var;
5098         } else {
5099                 field_var = NULL;
5100                 /*
5101                  * If no explicit system.event is specfied, default to
5102                  * looking for fields on the onmatch(system.event.xxx)
5103                  * event.
5104                  */
5105                 if (!system && data->handler == HANDLER_ONMATCH) {
5106                         system = data->match_data.event_system;
5107                         event = data->match_data.event;
5108                 }
5109
5110                 /*
5111                  * At this point, we're looking at a field on another
5112                  * event.  Because we can't modify a hist trigger on
5113                  * another event to add a variable for a field, we need
5114                  * to create a new trigger on that event and create the
5115                  * variable at the same time.
5116                  */
5117                 hist_field = create_field_var_hist(hist_data, system, event, var);
5118                 if (IS_ERR(hist_field))
5119                         goto free;
5120         }
5121  out:
5122         return hist_field;
5123  free:
5124         destroy_field_var(field_var);
5125         hist_field = NULL;
5126         goto out;
5127 }
5128
5129 static int trace_action_create(struct hist_trigger_data *hist_data,
5130                                struct action_data *data)
5131 {
5132         struct trace_array *tr = hist_data->event_file->tr;
5133         char *event_name, *param, *system = NULL;
5134         struct hist_field *hist_field, *var_ref;
5135         unsigned int i;
5136         unsigned int field_pos = 0;
5137         struct synth_event *event;
5138         char *synth_event_name;
5139         int var_ref_idx, ret = 0;
5140
5141         lockdep_assert_held(&event_mutex);
5142
5143         if (data->use_trace_keyword)
5144                 synth_event_name = data->synth_event_name;
5145         else
5146                 synth_event_name = data->action_name;
5147
5148         event = find_synth_event(synth_event_name);
5149         if (!event) {
5150                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
5151                 return -EINVAL;
5152         }
5153
5154         event->ref++;
5155
5156         for (i = 0; i < data->n_params; i++) {
5157                 char *p;
5158
5159                 p = param = kstrdup(data->params[i], GFP_KERNEL);
5160                 if (!param) {
5161                         ret = -ENOMEM;
5162                         goto err;
5163                 }
5164
5165                 system = strsep(&param, ".");
5166                 if (!param) {
5167                         param = (char *)system;
5168                         system = event_name = NULL;
5169                 } else {
5170                         event_name = strsep(&param, ".");
5171                         if (!param) {
5172                                 kfree(p);
5173                                 ret = -EINVAL;
5174                                 goto err;
5175                         }
5176                 }
5177
5178                 if (param[0] == '$')
5179                         hist_field = trace_action_find_var(hist_data, data,
5180                                                            system, event_name,
5181                                                            param);
5182                 else
5183                         hist_field = trace_action_create_field_var(hist_data,
5184                                                                    data,
5185                                                                    system,
5186                                                                    event_name,
5187                                                                    param);
5188
5189                 if (!hist_field) {
5190                         kfree(p);
5191                         ret = -EINVAL;
5192                         goto err;
5193                 }
5194
5195                 if (check_synth_field(event, hist_field, field_pos) == 0) {
5196                         var_ref = create_var_ref(hist_data, hist_field,
5197                                                  system, event_name);
5198                         if (!var_ref) {
5199                                 kfree(p);
5200                                 ret = -ENOMEM;
5201                                 goto err;
5202                         }
5203
5204                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
5205                         if (WARN_ON(var_ref_idx < 0)) {
5206                                 ret = var_ref_idx;
5207                                 goto err;
5208                         }
5209
5210                         data->var_ref_idx[i] = var_ref_idx;
5211
5212                         field_pos++;
5213                         kfree(p);
5214                         continue;
5215                 }
5216
5217                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
5218                 kfree(p);
5219                 ret = -EINVAL;
5220                 goto err;
5221         }
5222
5223         if (field_pos != event->n_fields) {
5224                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
5225                 ret = -EINVAL;
5226                 goto err;
5227         }
5228
5229         data->synth_event = event;
5230  out:
5231         return ret;
5232  err:
5233         event->ref--;
5234
5235         goto out;
5236 }
5237
5238 static int action_create(struct hist_trigger_data *hist_data,
5239                          struct action_data *data)
5240 {
5241         struct trace_event_file *file = hist_data->event_file;
5242         struct trace_array *tr = file->tr;
5243         struct track_data *track_data;
5244         struct field_var *field_var;
5245         unsigned int i;
5246         char *param;
5247         int ret = 0;
5248
5249         if (data->action == ACTION_TRACE)
5250                 return trace_action_create(hist_data, data);
5251
5252         if (data->action == ACTION_SNAPSHOT) {
5253                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
5254                 if (IS_ERR(track_data)) {
5255                         ret = PTR_ERR(track_data);
5256                         goto out;
5257                 }
5258
5259                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
5260                                                    cond_snapshot_update);
5261                 if (ret)
5262                         track_data_free(track_data);
5263
5264                 goto out;
5265         }
5266
5267         if (data->action == ACTION_SAVE) {
5268                 if (hist_data->n_save_vars) {
5269                         ret = -EEXIST;
5270                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
5271                         goto out;
5272                 }
5273
5274                 for (i = 0; i < data->n_params; i++) {
5275                         param = kstrdup(data->params[i], GFP_KERNEL);
5276                         if (!param) {
5277                                 ret = -ENOMEM;
5278                                 goto out;
5279                         }
5280
5281                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
5282                         if (IS_ERR(field_var)) {
5283                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
5284                                          errpos(param));
5285                                 ret = PTR_ERR(field_var);
5286                                 kfree(param);
5287                                 goto out;
5288                         }
5289
5290                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
5291                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
5292                                 hist_data->n_save_var_str++;
5293                         kfree(param);
5294                 }
5295         }
5296  out:
5297         return ret;
5298 }
5299
5300 static int onmatch_create(struct hist_trigger_data *hist_data,
5301                           struct action_data *data)
5302 {
5303         return action_create(hist_data, data);
5304 }
5305
5306 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
5307 {
5308         char *match_event, *match_event_system;
5309         struct action_data *data;
5310         int ret = -EINVAL;
5311
5312         data = kzalloc(sizeof(*data), GFP_KERNEL);
5313         if (!data)
5314                 return ERR_PTR(-ENOMEM);
5315
5316         match_event = strsep(&str, ")");
5317         if (!match_event || !str) {
5318                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
5319                 goto free;
5320         }
5321
5322         match_event_system = strsep(&match_event, ".");
5323         if (!match_event) {
5324                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
5325                 goto free;
5326         }
5327
5328         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
5329                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
5330                 goto free;
5331         }
5332
5333         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
5334         if (!data->match_data.event) {
5335                 ret = -ENOMEM;
5336                 goto free;
5337         }
5338
5339         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
5340         if (!data->match_data.event_system) {
5341                 ret = -ENOMEM;
5342                 goto free;
5343         }
5344
5345         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
5346         if (ret)
5347                 goto free;
5348  out:
5349         return data;
5350  free:
5351         onmatch_destroy(data);
5352         data = ERR_PTR(ret);
5353         goto out;
5354 }
5355
5356 static int create_hitcount_val(struct hist_trigger_data *hist_data)
5357 {
5358         hist_data->fields[HITCOUNT_IDX] =
5359                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
5360         if (!hist_data->fields[HITCOUNT_IDX])
5361                 return -ENOMEM;
5362
5363         hist_data->n_vals++;
5364         hist_data->n_fields++;
5365
5366         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
5367                 return -EINVAL;
5368
5369         return 0;
5370 }
5371
5372 static int __create_val_field(struct hist_trigger_data *hist_data,
5373                               unsigned int val_idx,
5374                               struct trace_event_file *file,
5375                               char *var_name, char *field_str,
5376                               unsigned long flags)
5377 {
5378         struct hist_field *hist_field;
5379         int ret = 0;
5380
5381         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
5382         if (IS_ERR(hist_field)) {
5383                 ret = PTR_ERR(hist_field);
5384                 goto out;
5385         }
5386
5387         hist_data->fields[val_idx] = hist_field;
5388
5389         ++hist_data->n_vals;
5390         ++hist_data->n_fields;
5391
5392         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
5393                 ret = -EINVAL;
5394  out:
5395         return ret;
5396 }
5397
5398 static int create_val_field(struct hist_trigger_data *hist_data,
5399                             unsigned int val_idx,
5400                             struct trace_event_file *file,
5401                             char *field_str)
5402 {
5403         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
5404                 return -EINVAL;
5405
5406         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
5407 }
5408
5409 static int create_var_field(struct hist_trigger_data *hist_data,
5410                             unsigned int val_idx,
5411                             struct trace_event_file *file,
5412                             char *var_name, char *expr_str)
5413 {
5414         struct trace_array *tr = hist_data->event_file->tr;
5415         unsigned long flags = 0;
5416
5417         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
5418                 return -EINVAL;
5419
5420         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
5421                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
5422                 return -EINVAL;
5423         }
5424
5425         flags |= HIST_FIELD_FL_VAR;
5426         hist_data->n_vars++;
5427         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
5428                 return -EINVAL;
5429
5430         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
5431 }
5432
5433 static int create_val_fields(struct hist_trigger_data *hist_data,
5434                              struct trace_event_file *file)
5435 {
5436         char *fields_str, *field_str;
5437         unsigned int i, j = 1;
5438         int ret;
5439
5440         ret = create_hitcount_val(hist_data);
5441         if (ret)
5442                 goto out;
5443
5444         fields_str = hist_data->attrs->vals_str;
5445         if (!fields_str)
5446                 goto out;
5447
5448         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
5449                      j < TRACING_MAP_VALS_MAX; i++) {
5450                 field_str = strsep(&fields_str, ",");
5451                 if (!field_str)
5452                         break;
5453
5454                 if (strcmp(field_str, "hitcount") == 0)
5455                         continue;
5456
5457                 ret = create_val_field(hist_data, j++, file, field_str);
5458                 if (ret)
5459                         goto out;
5460         }
5461
5462         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
5463                 ret = -EINVAL;
5464  out:
5465         return ret;
5466 }
5467
5468 static int create_key_field(struct hist_trigger_data *hist_data,
5469                             unsigned int key_idx,
5470                             unsigned int key_offset,
5471                             struct trace_event_file *file,
5472                             char *field_str)
5473 {
5474         struct trace_array *tr = hist_data->event_file->tr;
5475         struct hist_field *hist_field = NULL;
5476         unsigned long flags = 0;
5477         unsigned int key_size;
5478         int ret = 0;
5479
5480         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
5481                 return -EINVAL;
5482
5483         flags |= HIST_FIELD_FL_KEY;
5484
5485         if (strcmp(field_str, "stacktrace") == 0) {
5486                 flags |= HIST_FIELD_FL_STACKTRACE;
5487                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
5488                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
5489         } else {
5490                 hist_field = parse_expr(hist_data, file, field_str, flags,
5491                                         NULL, 0);
5492                 if (IS_ERR(hist_field)) {
5493                         ret = PTR_ERR(hist_field);
5494                         goto out;
5495                 }
5496
5497                 if (field_has_hist_vars(hist_field, 0)) {
5498                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
5499                         destroy_hist_field(hist_field, 0);
5500                         ret = -EINVAL;
5501                         goto out;
5502                 }
5503
5504                 key_size = hist_field->size;
5505         }
5506
5507         hist_data->fields[key_idx] = hist_field;
5508
5509         key_size = ALIGN(key_size, sizeof(u64));
5510         hist_data->fields[key_idx]->size = key_size;
5511         hist_data->fields[key_idx]->offset = key_offset;
5512
5513         hist_data->key_size += key_size;
5514
5515         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
5516                 ret = -EINVAL;
5517                 goto out;
5518         }
5519
5520         hist_data->n_keys++;
5521         hist_data->n_fields++;
5522
5523         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
5524                 return -EINVAL;
5525
5526         ret = key_size;
5527  out:
5528         return ret;
5529 }
5530
5531 static int create_key_fields(struct hist_trigger_data *hist_data,
5532                              struct trace_event_file *file)
5533 {
5534         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
5535         char *fields_str, *field_str;
5536         int ret = -EINVAL;
5537
5538         fields_str = hist_data->attrs->keys_str;
5539         if (!fields_str)
5540                 goto out;
5541
5542         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
5543                 field_str = strsep(&fields_str, ",");
5544                 if (!field_str)
5545                         break;
5546                 ret = create_key_field(hist_data, i, key_offset,
5547                                        file, field_str);
5548                 if (ret < 0)
5549                         goto out;
5550                 key_offset += ret;
5551         }
5552         if (fields_str) {
5553                 ret = -EINVAL;
5554                 goto out;
5555         }
5556         ret = 0;
5557  out:
5558         return ret;
5559 }
5560
5561 static int create_var_fields(struct hist_trigger_data *hist_data,
5562                              struct trace_event_file *file)
5563 {
5564         unsigned int i, j = hist_data->n_vals;
5565         int ret = 0;
5566
5567         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
5568
5569         for (i = 0; i < n_vars; i++) {
5570                 char *var_name = hist_data->attrs->var_defs.name[i];
5571                 char *expr = hist_data->attrs->var_defs.expr[i];
5572
5573                 ret = create_var_field(hist_data, j++, file, var_name, expr);
5574                 if (ret)
5575                         goto out;
5576         }
5577  out:
5578         return ret;
5579 }
5580
5581 static void free_var_defs(struct hist_trigger_data *hist_data)
5582 {
5583         unsigned int i;
5584
5585         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
5586                 kfree(hist_data->attrs->var_defs.name[i]);
5587                 kfree(hist_data->attrs->var_defs.expr[i]);
5588         }
5589
5590         hist_data->attrs->var_defs.n_vars = 0;
5591 }
5592
5593 static int parse_var_defs(struct hist_trigger_data *hist_data)
5594 {
5595         struct trace_array *tr = hist_data->event_file->tr;
5596         char *s, *str, *var_name, *field_str;
5597         unsigned int i, j, n_vars = 0;
5598         int ret = 0;
5599
5600         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
5601                 str = hist_data->attrs->assignment_str[i];
5602                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
5603                         field_str = strsep(&str, ",");
5604                         if (!field_str)
5605                                 break;
5606
5607                         var_name = strsep(&field_str, "=");
5608                         if (!var_name || !field_str) {
5609                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
5610                                          errpos(var_name));
5611                                 ret = -EINVAL;
5612                                 goto free;
5613                         }
5614
5615                         if (n_vars == TRACING_MAP_VARS_MAX) {
5616                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
5617                                 ret = -EINVAL;
5618                                 goto free;
5619                         }
5620
5621                         s = kstrdup(var_name, GFP_KERNEL);
5622                         if (!s) {
5623                                 ret = -ENOMEM;
5624                                 goto free;
5625                         }
5626                         hist_data->attrs->var_defs.name[n_vars] = s;
5627
5628                         s = kstrdup(field_str, GFP_KERNEL);
5629                         if (!s) {
5630                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
5631                                 ret = -ENOMEM;
5632                                 goto free;
5633                         }
5634                         hist_data->attrs->var_defs.expr[n_vars++] = s;
5635
5636                         hist_data->attrs->var_defs.n_vars = n_vars;
5637                 }
5638         }
5639
5640         return ret;
5641  free:
5642         free_var_defs(hist_data);
5643
5644         return ret;
5645 }
5646
5647 static int create_hist_fields(struct hist_trigger_data *hist_data,
5648                               struct trace_event_file *file)
5649 {
5650         int ret;
5651
5652         ret = parse_var_defs(hist_data);
5653         if (ret)
5654                 goto out;
5655
5656         ret = create_val_fields(hist_data, file);
5657         if (ret)
5658                 goto out;
5659
5660         ret = create_var_fields(hist_data, file);
5661         if (ret)
5662                 goto out;
5663
5664         ret = create_key_fields(hist_data, file);
5665         if (ret)
5666                 goto out;
5667  out:
5668         free_var_defs(hist_data);
5669
5670         return ret;
5671 }
5672
5673 static int is_descending(struct trace_array *tr, const char *str)
5674 {
5675         if (!str)
5676                 return 0;
5677
5678         if (strcmp(str, "descending") == 0)
5679                 return 1;
5680
5681         if (strcmp(str, "ascending") == 0)
5682                 return 0;
5683
5684         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
5685
5686         return -EINVAL;
5687 }
5688
5689 static int create_sort_keys(struct hist_trigger_data *hist_data)
5690 {
5691         struct trace_array *tr = hist_data->event_file->tr;
5692         char *fields_str = hist_data->attrs->sort_key_str;
5693         struct tracing_map_sort_key *sort_key;
5694         int descending, ret = 0;
5695         unsigned int i, j, k;
5696
5697         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
5698
5699         if (!fields_str)
5700                 goto out;
5701
5702         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
5703                 struct hist_field *hist_field;
5704                 char *field_str, *field_name;
5705                 const char *test_name;
5706
5707                 sort_key = &hist_data->sort_keys[i];
5708
5709                 field_str = strsep(&fields_str, ",");
5710                 if (!field_str)
5711                         break;
5712
5713                 if (!*field_str) {
5714                         ret = -EINVAL;
5715                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
5716                         break;
5717                 }
5718
5719                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
5720                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
5721                         ret = -EINVAL;
5722                         break;
5723                 }
5724
5725                 field_name = strsep(&field_str, ".");
5726                 if (!field_name || !*field_name) {
5727                         ret = -EINVAL;
5728                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
5729                         break;
5730                 }
5731
5732                 if (strcmp(field_name, "hitcount") == 0) {
5733                         descending = is_descending(tr, field_str);
5734                         if (descending < 0) {
5735                                 ret = descending;
5736                                 break;
5737                         }
5738                         sort_key->descending = descending;
5739                         continue;
5740                 }
5741
5742                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
5743                         unsigned int idx;
5744
5745                         hist_field = hist_data->fields[j];
5746                         if (hist_field->flags & HIST_FIELD_FL_VAR)
5747                                 continue;
5748
5749                         idx = k++;
5750
5751                         test_name = hist_field_name(hist_field, 0);
5752
5753                         if (strcmp(field_name, test_name) == 0) {
5754                                 sort_key->field_idx = idx;
5755                                 descending = is_descending(tr, field_str);
5756                                 if (descending < 0) {
5757                                         ret = descending;
5758                                         goto out;
5759                                 }
5760                                 sort_key->descending = descending;
5761                                 break;
5762                         }
5763                 }
5764                 if (j == hist_data->n_fields) {
5765                         ret = -EINVAL;
5766                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
5767                         break;
5768                 }
5769         }
5770
5771         hist_data->n_sort_keys = i;
5772  out:
5773         return ret;
5774 }
5775
5776 static void destroy_actions(struct hist_trigger_data *hist_data)
5777 {
5778         unsigned int i;
5779
5780         for (i = 0; i < hist_data->n_actions; i++) {
5781                 struct action_data *data = hist_data->actions[i];
5782
5783                 if (data->handler == HANDLER_ONMATCH)
5784                         onmatch_destroy(data);
5785                 else if (data->handler == HANDLER_ONMAX ||
5786                          data->handler == HANDLER_ONCHANGE)
5787                         track_data_destroy(hist_data, data);
5788                 else
5789                         kfree(data);
5790         }
5791 }
5792
5793 static int parse_actions(struct hist_trigger_data *hist_data)
5794 {
5795         struct trace_array *tr = hist_data->event_file->tr;
5796         struct action_data *data;
5797         unsigned int i;
5798         int ret = 0;
5799         char *str;
5800         int len;
5801
5802         for (i = 0; i < hist_data->attrs->n_actions; i++) {
5803                 str = hist_data->attrs->action_str[i];
5804
5805                 if ((len = str_has_prefix(str, "onmatch("))) {
5806                         char *action_str = str + len;
5807
5808                         data = onmatch_parse(tr, action_str);
5809                         if (IS_ERR(data)) {
5810                                 ret = PTR_ERR(data);
5811                                 break;
5812                         }
5813                 } else if ((len = str_has_prefix(str, "onmax("))) {
5814                         char *action_str = str + len;
5815
5816                         data = track_data_parse(hist_data, action_str,
5817                                                 HANDLER_ONMAX);
5818                         if (IS_ERR(data)) {
5819                                 ret = PTR_ERR(data);
5820                                 break;
5821                         }
5822                 } else if ((len = str_has_prefix(str, "onchange("))) {
5823                         char *action_str = str + len;
5824
5825                         data = track_data_parse(hist_data, action_str,
5826                                                 HANDLER_ONCHANGE);
5827                         if (IS_ERR(data)) {
5828                                 ret = PTR_ERR(data);
5829                                 break;
5830                         }
5831                 } else {
5832                         ret = -EINVAL;
5833                         break;
5834                 }
5835
5836                 hist_data->actions[hist_data->n_actions++] = data;
5837         }
5838
5839         return ret;
5840 }
5841
5842 static int create_actions(struct hist_trigger_data *hist_data)
5843 {
5844         struct action_data *data;
5845         unsigned int i;
5846         int ret = 0;
5847
5848         for (i = 0; i < hist_data->attrs->n_actions; i++) {
5849                 data = hist_data->actions[i];
5850
5851                 if (data->handler == HANDLER_ONMATCH) {
5852                         ret = onmatch_create(hist_data, data);
5853                         if (ret)
5854                                 break;
5855                 } else if (data->handler == HANDLER_ONMAX ||
5856                            data->handler == HANDLER_ONCHANGE) {
5857                         ret = track_data_create(hist_data, data);
5858                         if (ret)
5859                                 break;
5860                 } else {
5861                         ret = -EINVAL;
5862                         break;
5863                 }
5864         }
5865
5866         return ret;
5867 }
5868
5869 static void print_actions(struct seq_file *m,
5870                           struct hist_trigger_data *hist_data,
5871                           struct tracing_map_elt *elt)
5872 {
5873         unsigned int i;
5874
5875         for (i = 0; i < hist_data->n_actions; i++) {
5876                 struct action_data *data = hist_data->actions[i];
5877
5878                 if (data->action == ACTION_SNAPSHOT)
5879                         continue;
5880
5881                 if (data->handler == HANDLER_ONMAX ||
5882                     data->handler == HANDLER_ONCHANGE)
5883                         track_data_print(m, hist_data, elt, data);
5884         }
5885 }
5886
5887 static void print_action_spec(struct seq_file *m,
5888                               struct hist_trigger_data *hist_data,
5889                               struct action_data *data)
5890 {
5891         unsigned int i;
5892
5893         if (data->action == ACTION_SAVE) {
5894                 for (i = 0; i < hist_data->n_save_vars; i++) {
5895                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
5896                         if (i < hist_data->n_save_vars - 1)
5897                                 seq_puts(m, ",");
5898                 }
5899         } else if (data->action == ACTION_TRACE) {
5900                 if (data->use_trace_keyword)
5901                         seq_printf(m, "%s", data->synth_event_name);
5902                 for (i = 0; i < data->n_params; i++) {
5903                         if (i || data->use_trace_keyword)
5904                                 seq_puts(m, ",");
5905                         seq_printf(m, "%s", data->params[i]);
5906                 }
5907         }
5908 }
5909
5910 static void print_track_data_spec(struct seq_file *m,
5911                                   struct hist_trigger_data *hist_data,
5912                                   struct action_data *data)
5913 {
5914         if (data->handler == HANDLER_ONMAX)
5915                 seq_puts(m, ":onmax(");
5916         else if (data->handler == HANDLER_ONCHANGE)
5917                 seq_puts(m, ":onchange(");
5918         seq_printf(m, "%s", data->track_data.var_str);
5919         seq_printf(m, ").%s(", data->action_name);
5920
5921         print_action_spec(m, hist_data, data);
5922
5923         seq_puts(m, ")");
5924 }
5925
5926 static void print_onmatch_spec(struct seq_file *m,
5927                                struct hist_trigger_data *hist_data,
5928                                struct action_data *data)
5929 {
5930         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
5931                    data->match_data.event);
5932
5933         seq_printf(m, "%s(", data->action_name);
5934
5935         print_action_spec(m, hist_data, data);
5936
5937         seq_puts(m, ")");
5938 }
5939
5940 static bool actions_match(struct hist_trigger_data *hist_data,
5941                           struct hist_trigger_data *hist_data_test)
5942 {
5943         unsigned int i, j;
5944
5945         if (hist_data->n_actions != hist_data_test->n_actions)
5946                 return false;
5947
5948         for (i = 0; i < hist_data->n_actions; i++) {
5949                 struct action_data *data = hist_data->actions[i];
5950                 struct action_data *data_test = hist_data_test->actions[i];
5951                 char *action_name, *action_name_test;
5952
5953                 if (data->handler != data_test->handler)
5954                         return false;
5955                 if (data->action != data_test->action)
5956                         return false;
5957
5958                 if (data->n_params != data_test->n_params)
5959                         return false;
5960
5961                 for (j = 0; j < data->n_params; j++) {
5962                         if (strcmp(data->params[j], data_test->params[j]) != 0)
5963                                 return false;
5964                 }
5965
5966                 if (data->use_trace_keyword)
5967                         action_name = data->synth_event_name;
5968                 else
5969                         action_name = data->action_name;
5970
5971                 if (data_test->use_trace_keyword)
5972                         action_name_test = data_test->synth_event_name;
5973                 else
5974                         action_name_test = data_test->action_name;
5975
5976                 if (strcmp(action_name, action_name_test) != 0)
5977                         return false;
5978
5979                 if (data->handler == HANDLER_ONMATCH) {
5980                         if (strcmp(data->match_data.event_system,
5981                                    data_test->match_data.event_system) != 0)
5982                                 return false;
5983                         if (strcmp(data->match_data.event,
5984                                    data_test->match_data.event) != 0)
5985                                 return false;
5986                 } else if (data->handler == HANDLER_ONMAX ||
5987                            data->handler == HANDLER_ONCHANGE) {
5988                         if (strcmp(data->track_data.var_str,
5989                                    data_test->track_data.var_str) != 0)
5990                                 return false;
5991                 }
5992         }
5993
5994         return true;
5995 }
5996
5997
5998 static void print_actions_spec(struct seq_file *m,
5999                                struct hist_trigger_data *hist_data)
6000 {
6001         unsigned int i;
6002
6003         for (i = 0; i < hist_data->n_actions; i++) {
6004                 struct action_data *data = hist_data->actions[i];
6005
6006                 if (data->handler == HANDLER_ONMATCH)
6007                         print_onmatch_spec(m, hist_data, data);
6008                 else if (data->handler == HANDLER_ONMAX ||
6009                          data->handler == HANDLER_ONCHANGE)
6010                         print_track_data_spec(m, hist_data, data);
6011         }
6012 }
6013
6014 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
6015 {
6016         unsigned int i;
6017
6018         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6019                 kfree(hist_data->field_var_hists[i]->cmd);
6020                 kfree(hist_data->field_var_hists[i]);
6021         }
6022 }
6023
6024 static void destroy_hist_data(struct hist_trigger_data *hist_data)
6025 {
6026         if (!hist_data)
6027                 return;
6028
6029         destroy_hist_trigger_attrs(hist_data->attrs);
6030         destroy_hist_fields(hist_data);
6031         tracing_map_destroy(hist_data->map);
6032
6033         destroy_actions(hist_data);
6034         destroy_field_vars(hist_data);
6035         destroy_field_var_hists(hist_data);
6036
6037         kfree(hist_data);
6038 }
6039
6040 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
6041 {
6042         struct tracing_map *map = hist_data->map;
6043         struct ftrace_event_field *field;
6044         struct hist_field *hist_field;
6045         int i, idx = 0;
6046
6047         for_each_hist_field(i, hist_data) {
6048                 hist_field = hist_data->fields[i];
6049                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
6050                         tracing_map_cmp_fn_t cmp_fn;
6051
6052                         field = hist_field->field;
6053
6054                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
6055                                 cmp_fn = tracing_map_cmp_none;
6056                         else if (!field)
6057                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
6058                                                              hist_field->is_signed);
6059                         else if (is_string_field(field))
6060                                 cmp_fn = tracing_map_cmp_string;
6061                         else
6062                                 cmp_fn = tracing_map_cmp_num(field->size,
6063                                                              field->is_signed);
6064                         idx = tracing_map_add_key_field(map,
6065                                                         hist_field->offset,
6066                                                         cmp_fn);
6067                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
6068                         idx = tracing_map_add_sum_field(map);
6069
6070                 if (idx < 0)
6071                         return idx;
6072
6073                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6074                         idx = tracing_map_add_var(map);
6075                         if (idx < 0)
6076                                 return idx;
6077                         hist_field->var.idx = idx;
6078                         hist_field->var.hist_data = hist_data;
6079                 }
6080         }
6081
6082         return 0;
6083 }
6084
6085 static struct hist_trigger_data *
6086 create_hist_data(unsigned int map_bits,
6087                  struct hist_trigger_attrs *attrs,
6088                  struct trace_event_file *file,
6089                  bool remove)
6090 {
6091         const struct tracing_map_ops *map_ops = NULL;
6092         struct hist_trigger_data *hist_data;
6093         int ret = 0;
6094
6095         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
6096         if (!hist_data)
6097                 return ERR_PTR(-ENOMEM);
6098
6099         hist_data->attrs = attrs;
6100         hist_data->remove = remove;
6101         hist_data->event_file = file;
6102
6103         ret = parse_actions(hist_data);
6104         if (ret)
6105                 goto free;
6106
6107         ret = create_hist_fields(hist_data, file);
6108         if (ret)
6109                 goto free;
6110
6111         ret = create_sort_keys(hist_data);
6112         if (ret)
6113                 goto free;
6114
6115         map_ops = &hist_trigger_elt_data_ops;
6116
6117         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6118                                             map_ops, hist_data);
6119         if (IS_ERR(hist_data->map)) {
6120                 ret = PTR_ERR(hist_data->map);
6121                 hist_data->map = NULL;
6122                 goto free;
6123         }
6124
6125         ret = create_tracing_map_fields(hist_data);
6126         if (ret)
6127                 goto free;
6128  out:
6129         return hist_data;
6130  free:
6131         hist_data->attrs = NULL;
6132
6133         destroy_hist_data(hist_data);
6134
6135         hist_data = ERR_PTR(ret);
6136
6137         goto out;
6138 }
6139
6140 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
6141                                     struct tracing_map_elt *elt, void *rec,
6142                                     struct ring_buffer_event *rbe,
6143                                     u64 *var_ref_vals)
6144 {
6145         struct hist_elt_data *elt_data;
6146         struct hist_field *hist_field;
6147         unsigned int i, var_idx;
6148         u64 hist_val;
6149
6150         elt_data = elt->private_data;
6151         elt_data->var_ref_vals = var_ref_vals;
6152
6153         for_each_hist_val_field(i, hist_data) {
6154                 hist_field = hist_data->fields[i];
6155                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
6156                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6157                         var_idx = hist_field->var.idx;
6158                         tracing_map_set_var(elt, var_idx, hist_val);
6159                         continue;
6160                 }
6161                 tracing_map_update_sum(elt, i, hist_val);
6162         }
6163
6164         for_each_hist_key_field(i, hist_data) {
6165                 hist_field = hist_data->fields[i];
6166                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
6167                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
6168                         var_idx = hist_field->var.idx;
6169                         tracing_map_set_var(elt, var_idx, hist_val);
6170                 }
6171         }
6172
6173         update_field_vars(hist_data, elt, rbe, rec);
6174 }
6175
6176 static inline void add_to_key(char *compound_key, void *key,
6177                               struct hist_field *key_field, void *rec)
6178 {
6179         size_t size = key_field->size;
6180
6181         if (key_field->flags & HIST_FIELD_FL_STRING) {
6182                 struct ftrace_event_field *field;
6183
6184                 field = key_field->field;
6185                 if (field->filter_type == FILTER_DYN_STRING)
6186                         size = *(u32 *)(rec + field->offset) >> 16;
6187                 else if (field->filter_type == FILTER_PTR_STRING)
6188                         size = strlen(key);
6189                 else if (field->filter_type == FILTER_STATIC_STRING)
6190                         size = field->size;
6191
6192                 /* ensure NULL-termination */
6193                 if (size > key_field->size - 1)
6194                         size = key_field->size - 1;
6195
6196                 strncpy(compound_key + key_field->offset, (char *)key, size);
6197         } else
6198                 memcpy(compound_key + key_field->offset, key, size);
6199 }
6200
6201 static void
6202 hist_trigger_actions(struct hist_trigger_data *hist_data,
6203                      struct tracing_map_elt *elt, void *rec,
6204                      struct ring_buffer_event *rbe, void *key,
6205                      u64 *var_ref_vals)
6206 {
6207         struct action_data *data;
6208         unsigned int i;
6209
6210         for (i = 0; i < hist_data->n_actions; i++) {
6211                 data = hist_data->actions[i];
6212                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
6213         }
6214 }
6215
6216 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
6217                                struct ring_buffer_event *rbe)
6218 {
6219         struct hist_trigger_data *hist_data = data->private_data;
6220         bool use_compound_key = (hist_data->n_keys > 1);
6221         unsigned long entries[HIST_STACKTRACE_DEPTH];
6222         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
6223         char compound_key[HIST_KEY_SIZE_MAX];
6224         struct tracing_map_elt *elt = NULL;
6225         struct hist_field *key_field;
6226         u64 field_contents;
6227         void *key = NULL;
6228         unsigned int i;
6229
6230         memset(compound_key, 0, hist_data->key_size);
6231
6232         for_each_hist_key_field(i, hist_data) {
6233                 key_field = hist_data->fields[i];
6234
6235                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
6236                         memset(entries, 0, HIST_STACKTRACE_SIZE);
6237                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
6238                                          HIST_STACKTRACE_SKIP);
6239                         key = entries;
6240                 } else {
6241                         field_contents = key_field->fn(key_field, elt, rbe, rec);
6242                         if (key_field->flags & HIST_FIELD_FL_STRING) {
6243                                 key = (void *)(unsigned long)field_contents;
6244                                 use_compound_key = true;
6245                         } else
6246                                 key = (void *)&field_contents;
6247                 }
6248
6249                 if (use_compound_key)
6250                         add_to_key(compound_key, key, key_field, rec);
6251         }
6252
6253         if (use_compound_key)
6254                 key = compound_key;
6255
6256         if (hist_data->n_var_refs &&
6257             !resolve_var_refs(hist_data, key, var_ref_vals, false))
6258                 return;
6259
6260         elt = tracing_map_insert(hist_data->map, key);
6261         if (!elt)
6262                 return;
6263
6264         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
6265
6266         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
6267                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
6268 }
6269
6270 static void hist_trigger_stacktrace_print(struct seq_file *m,
6271                                           unsigned long *stacktrace_entries,
6272                                           unsigned int max_entries)
6273 {
6274         char str[KSYM_SYMBOL_LEN];
6275         unsigned int spaces = 8;
6276         unsigned int i;
6277
6278         for (i = 0; i < max_entries; i++) {
6279                 if (!stacktrace_entries[i])
6280                         return;
6281
6282                 seq_printf(m, "%*c", 1 + spaces, ' ');
6283                 sprint_symbol(str, stacktrace_entries[i]);
6284                 seq_printf(m, "%s\n", str);
6285         }
6286 }
6287
6288 static void hist_trigger_print_key(struct seq_file *m,
6289                                    struct hist_trigger_data *hist_data,
6290                                    void *key,
6291                                    struct tracing_map_elt *elt)
6292 {
6293         struct hist_field *key_field;
6294         char str[KSYM_SYMBOL_LEN];
6295         bool multiline = false;
6296         const char *field_name;
6297         unsigned int i;
6298         u64 uval;
6299
6300         seq_puts(m, "{ ");
6301
6302         for_each_hist_key_field(i, hist_data) {
6303                 key_field = hist_data->fields[i];
6304
6305                 if (i > hist_data->n_vals)
6306                         seq_puts(m, ", ");
6307
6308                 field_name = hist_field_name(key_field, 0);
6309
6310                 if (key_field->flags & HIST_FIELD_FL_HEX) {
6311                         uval = *(u64 *)(key + key_field->offset);
6312                         seq_printf(m, "%s: %llx", field_name, uval);
6313                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
6314                         uval = *(u64 *)(key + key_field->offset);
6315                         sprint_symbol_no_offset(str, uval);
6316                         seq_printf(m, "%s: [%llx] %-45s", field_name,
6317                                    uval, str);
6318                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
6319                         uval = *(u64 *)(key + key_field->offset);
6320                         sprint_symbol(str, uval);
6321                         seq_printf(m, "%s: [%llx] %-55s", field_name,
6322                                    uval, str);
6323                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
6324                         struct hist_elt_data *elt_data = elt->private_data;
6325                         char *comm;
6326
6327                         if (WARN_ON_ONCE(!elt_data))
6328                                 return;
6329
6330                         comm = elt_data->comm;
6331
6332                         uval = *(u64 *)(key + key_field->offset);
6333                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
6334                                    comm, uval);
6335                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
6336                         const char *syscall_name;
6337
6338                         uval = *(u64 *)(key + key_field->offset);
6339                         syscall_name = get_syscall_name(uval);
6340                         if (!syscall_name)
6341                                 syscall_name = "unknown_syscall";
6342
6343                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
6344                                    syscall_name, uval);
6345                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
6346                         seq_puts(m, "stacktrace:\n");
6347                         hist_trigger_stacktrace_print(m,
6348                                                       key + key_field->offset,
6349                                                       HIST_STACKTRACE_DEPTH);
6350                         multiline = true;
6351                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
6352                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
6353                                    *(u64 *)(key + key_field->offset));
6354                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
6355                         seq_printf(m, "%s: %-50s", field_name,
6356                                    (char *)(key + key_field->offset));
6357                 } else {
6358                         uval = *(u64 *)(key + key_field->offset);
6359                         seq_printf(m, "%s: %10llu", field_name, uval);
6360                 }
6361         }
6362
6363         if (!multiline)
6364                 seq_puts(m, " ");
6365
6366         seq_puts(m, "}");
6367 }
6368
6369 static void hist_trigger_entry_print(struct seq_file *m,
6370                                      struct hist_trigger_data *hist_data,
6371                                      void *key,
6372                                      struct tracing_map_elt *elt)
6373 {
6374         const char *field_name;
6375         unsigned int i;
6376
6377         hist_trigger_print_key(m, hist_data, key, elt);
6378
6379         seq_printf(m, " hitcount: %10llu",
6380                    tracing_map_read_sum(elt, HITCOUNT_IDX));
6381
6382         for (i = 1; i < hist_data->n_vals; i++) {
6383                 field_name = hist_field_name(hist_data->fields[i], 0);
6384
6385                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
6386                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
6387                         continue;
6388
6389                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
6390                         seq_printf(m, "  %s: %10llx", field_name,
6391                                    tracing_map_read_sum(elt, i));
6392                 } else {
6393                         seq_printf(m, "  %s: %10llu", field_name,
6394                                    tracing_map_read_sum(elt, i));
6395                 }
6396         }
6397
6398         print_actions(m, hist_data, elt);
6399
6400         seq_puts(m, "\n");
6401 }
6402
6403 static int print_entries(struct seq_file *m,
6404                          struct hist_trigger_data *hist_data)
6405 {
6406         struct tracing_map_sort_entry **sort_entries = NULL;
6407         struct tracing_map *map = hist_data->map;
6408         int i, n_entries;
6409
6410         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
6411                                              hist_data->n_sort_keys,
6412                                              &sort_entries);
6413         if (n_entries < 0)
6414                 return n_entries;
6415
6416         for (i = 0; i < n_entries; i++)
6417                 hist_trigger_entry_print(m, hist_data,
6418                                          sort_entries[i]->key,
6419                                          sort_entries[i]->elt);
6420
6421         tracing_map_destroy_sort_entries(sort_entries, n_entries);
6422
6423         return n_entries;
6424 }
6425
6426 static void hist_trigger_show(struct seq_file *m,
6427                               struct event_trigger_data *data, int n)
6428 {
6429         struct hist_trigger_data *hist_data;
6430         int n_entries;
6431
6432         if (n > 0)
6433                 seq_puts(m, "\n\n");
6434
6435         seq_puts(m, "# event histogram\n#\n# trigger info: ");
6436         data->ops->print(m, data->ops, data);
6437         seq_puts(m, "#\n\n");
6438
6439         hist_data = data->private_data;
6440         n_entries = print_entries(m, hist_data);
6441         if (n_entries < 0)
6442                 n_entries = 0;
6443
6444         track_data_snapshot_print(m, hist_data);
6445
6446         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
6447                    (u64)atomic64_read(&hist_data->map->hits),
6448                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
6449 }
6450
6451 static int hist_show(struct seq_file *m, void *v)
6452 {
6453         struct event_trigger_data *data;
6454         struct trace_event_file *event_file;
6455         int n = 0, ret = 0;
6456
6457         mutex_lock(&event_mutex);
6458
6459         event_file = event_file_data(m->private);
6460         if (unlikely(!event_file)) {
6461                 ret = -ENODEV;
6462                 goto out_unlock;
6463         }
6464
6465         list_for_each_entry(data, &event_file->triggers, list) {
6466                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
6467                         hist_trigger_show(m, data, n++);
6468         }
6469
6470  out_unlock:
6471         mutex_unlock(&event_mutex);
6472
6473         return ret;
6474 }
6475
6476 static int event_hist_open(struct inode *inode, struct file *file)
6477 {
6478         int ret;
6479
6480         ret = security_locked_down(LOCKDOWN_TRACEFS);
6481         if (ret)
6482                 return ret;
6483
6484         return single_open(file, hist_show, file);
6485 }
6486
6487 const struct file_operations event_hist_fops = {
6488         .open = event_hist_open,
6489         .read = seq_read,
6490         .llseek = seq_lseek,
6491         .release = single_release,
6492 };
6493
6494 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
6495 {
6496         const char *field_name = hist_field_name(hist_field, 0);
6497
6498         if (hist_field->var.name)
6499                 seq_printf(m, "%s=", hist_field->var.name);
6500
6501         if (hist_field->flags & HIST_FIELD_FL_CPU)
6502                 seq_puts(m, "cpu");
6503         else if (field_name) {
6504                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
6505                     hist_field->flags & HIST_FIELD_FL_ALIAS)
6506                         seq_putc(m, '$');
6507                 seq_printf(m, "%s", field_name);
6508         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
6509                 seq_puts(m, "common_timestamp");
6510
6511         if (hist_field->flags) {
6512                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
6513                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
6514                         const char *flags = get_hist_field_flags(hist_field);
6515
6516                         if (flags)
6517                                 seq_printf(m, ".%s", flags);
6518                 }
6519         }
6520 }
6521
6522 static int event_hist_trigger_print(struct seq_file *m,
6523                                     struct event_trigger_ops *ops,
6524                                     struct event_trigger_data *data)
6525 {
6526         struct hist_trigger_data *hist_data = data->private_data;
6527         struct hist_field *field;
6528         bool have_var = false;
6529         unsigned int i;
6530
6531         seq_puts(m, "hist:");
6532
6533         if (data->name)
6534                 seq_printf(m, "%s:", data->name);
6535
6536         seq_puts(m, "keys=");
6537
6538         for_each_hist_key_field(i, hist_data) {
6539                 field = hist_data->fields[i];
6540
6541                 if (i > hist_data->n_vals)
6542                         seq_puts(m, ",");
6543
6544                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
6545                         seq_puts(m, "stacktrace");
6546                 else
6547                         hist_field_print(m, field);
6548         }
6549
6550         seq_puts(m, ":vals=");
6551
6552         for_each_hist_val_field(i, hist_data) {
6553                 field = hist_data->fields[i];
6554                 if (field->flags & HIST_FIELD_FL_VAR) {
6555                         have_var = true;
6556                         continue;
6557                 }
6558
6559                 if (i == HITCOUNT_IDX)
6560                         seq_puts(m, "hitcount");
6561                 else {
6562                         seq_puts(m, ",");
6563                         hist_field_print(m, field);
6564                 }
6565         }
6566
6567         if (have_var) {
6568                 unsigned int n = 0;
6569
6570                 seq_puts(m, ":");
6571
6572                 for_each_hist_val_field(i, hist_data) {
6573                         field = hist_data->fields[i];
6574
6575                         if (field->flags & HIST_FIELD_FL_VAR) {
6576                                 if (n++)
6577                                         seq_puts(m, ",");
6578                                 hist_field_print(m, field);
6579                         }
6580                 }
6581         }
6582
6583         seq_puts(m, ":sort=");
6584
6585         for (i = 0; i < hist_data->n_sort_keys; i++) {
6586                 struct tracing_map_sort_key *sort_key;
6587                 unsigned int idx, first_key_idx;
6588
6589                 /* skip VAR vals */
6590                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6591
6592                 sort_key = &hist_data->sort_keys[i];
6593                 idx = sort_key->field_idx;
6594
6595                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6596                         return -EINVAL;
6597
6598                 if (i > 0)
6599                         seq_puts(m, ",");
6600
6601                 if (idx == HITCOUNT_IDX)
6602                         seq_puts(m, "hitcount");
6603                 else {
6604                         if (idx >= first_key_idx)
6605                                 idx += hist_data->n_vars;
6606                         hist_field_print(m, hist_data->fields[idx]);
6607                 }
6608
6609                 if (sort_key->descending)
6610                         seq_puts(m, ".descending");
6611         }
6612         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6613         if (hist_data->enable_timestamps)
6614                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6615
6616         print_actions_spec(m, hist_data);
6617
6618         if (data->filter_str)
6619                 seq_printf(m, " if %s", data->filter_str);
6620
6621         if (data->paused)
6622                 seq_puts(m, " [paused]");
6623         else
6624                 seq_puts(m, " [active]");
6625
6626         seq_putc(m, '\n');
6627
6628         return 0;
6629 }
6630
6631 static int event_hist_trigger_init(struct event_trigger_ops *ops,
6632                                    struct event_trigger_data *data)
6633 {
6634         struct hist_trigger_data *hist_data = data->private_data;
6635
6636         if (!data->ref && hist_data->attrs->name)
6637                 save_named_trigger(hist_data->attrs->name, data);
6638
6639         data->ref++;
6640
6641         return 0;
6642 }
6643
6644 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6645 {
6646         struct trace_event_file *file;
6647         unsigned int i;
6648         char *cmd;
6649         int ret;
6650
6651         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6652                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6653                 cmd = hist_data->field_var_hists[i]->cmd;
6654                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
6655                                               "!hist", "hist", cmd);
6656         }
6657 }
6658
6659 static void event_hist_trigger_free(struct event_trigger_ops *ops,
6660                                     struct event_trigger_data *data)
6661 {
6662         struct hist_trigger_data *hist_data = data->private_data;
6663
6664         if (WARN_ON_ONCE(data->ref <= 0))
6665                 return;
6666
6667         data->ref--;
6668         if (!data->ref) {
6669                 if (data->name)
6670                         del_named_trigger(data);
6671
6672                 trigger_data_free(data);
6673
6674                 remove_hist_vars(hist_data);
6675
6676                 unregister_field_var_hists(hist_data);
6677
6678                 destroy_hist_data(hist_data);
6679         }
6680 }
6681
6682 static struct event_trigger_ops event_hist_trigger_ops = {
6683         .func                   = event_hist_trigger,
6684         .print                  = event_hist_trigger_print,
6685         .init                   = event_hist_trigger_init,
6686         .free                   = event_hist_trigger_free,
6687 };
6688
6689 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
6690                                          struct event_trigger_data *data)
6691 {
6692         data->ref++;
6693
6694         save_named_trigger(data->named_data->name, data);
6695
6696         event_hist_trigger_init(ops, data->named_data);
6697
6698         return 0;
6699 }
6700
6701 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
6702                                           struct event_trigger_data *data)
6703 {
6704         if (WARN_ON_ONCE(data->ref <= 0))
6705                 return;
6706
6707         event_hist_trigger_free(ops, data->named_data);
6708
6709         data->ref--;
6710         if (!data->ref) {
6711                 del_named_trigger(data);
6712                 trigger_data_free(data);
6713         }
6714 }
6715
6716 static struct event_trigger_ops event_hist_trigger_named_ops = {
6717         .func                   = event_hist_trigger,
6718         .print                  = event_hist_trigger_print,
6719         .init                   = event_hist_trigger_named_init,
6720         .free                   = event_hist_trigger_named_free,
6721 };
6722
6723 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6724                                                             char *param)
6725 {
6726         return &event_hist_trigger_ops;
6727 }
6728
6729 static void hist_clear(struct event_trigger_data *data)
6730 {
6731         struct hist_trigger_data *hist_data = data->private_data;
6732
6733         if (data->name)
6734                 pause_named_trigger(data);
6735
6736         tracepoint_synchronize_unregister();
6737
6738         tracing_map_clear(hist_data->map);
6739
6740         if (data->name)
6741                 unpause_named_trigger(data);
6742 }
6743
6744 static bool compatible_field(struct ftrace_event_field *field,
6745                              struct ftrace_event_field *test_field)
6746 {
6747         if (field == test_field)
6748                 return true;
6749         if (field == NULL || test_field == NULL)
6750                 return false;
6751         if (strcmp(field->name, test_field->name) != 0)
6752                 return false;
6753         if (strcmp(field->type, test_field->type) != 0)
6754                 return false;
6755         if (field->size != test_field->size)
6756                 return false;
6757         if (field->is_signed != test_field->is_signed)
6758                 return false;
6759
6760         return true;
6761 }
6762
6763 static bool hist_trigger_match(struct event_trigger_data *data,
6764                                struct event_trigger_data *data_test,
6765                                struct event_trigger_data *named_data,
6766                                bool ignore_filter)
6767 {
6768         struct tracing_map_sort_key *sort_key, *sort_key_test;
6769         struct hist_trigger_data *hist_data, *hist_data_test;
6770         struct hist_field *key_field, *key_field_test;
6771         unsigned int i;
6772
6773         if (named_data && (named_data != data_test) &&
6774             (named_data != data_test->named_data))
6775                 return false;
6776
6777         if (!named_data && is_named_trigger(data_test))
6778                 return false;
6779
6780         hist_data = data->private_data;
6781         hist_data_test = data_test->private_data;
6782
6783         if (hist_data->n_vals != hist_data_test->n_vals ||
6784             hist_data->n_fields != hist_data_test->n_fields ||
6785             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6786                 return false;
6787
6788         if (!ignore_filter) {
6789                 if ((data->filter_str && !data_test->filter_str) ||
6790                    (!data->filter_str && data_test->filter_str))
6791                         return false;
6792         }
6793
6794         for_each_hist_field(i, hist_data) {
6795                 key_field = hist_data->fields[i];
6796                 key_field_test = hist_data_test->fields[i];
6797
6798                 if (key_field->flags != key_field_test->flags)
6799                         return false;
6800                 if (!compatible_field(key_field->field, key_field_test->field))
6801                         return false;
6802                 if (key_field->offset != key_field_test->offset)
6803                         return false;
6804                 if (key_field->size != key_field_test->size)
6805                         return false;
6806                 if (key_field->is_signed != key_field_test->is_signed)
6807                         return false;
6808                 if (!!key_field->var.name != !!key_field_test->var.name)
6809                         return false;
6810                 if (key_field->var.name &&
6811                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6812                         return false;
6813         }
6814
6815         for (i = 0; i < hist_data->n_sort_keys; i++) {
6816                 sort_key = &hist_data->sort_keys[i];
6817                 sort_key_test = &hist_data_test->sort_keys[i];
6818
6819                 if (sort_key->field_idx != sort_key_test->field_idx ||
6820                     sort_key->descending != sort_key_test->descending)
6821                         return false;
6822         }
6823
6824         if (!ignore_filter && data->filter_str &&
6825             (strcmp(data->filter_str, data_test->filter_str) != 0))
6826                 return false;
6827
6828         if (!actions_match(hist_data, hist_data_test))
6829                 return false;
6830
6831         return true;
6832 }
6833
6834 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
6835                                  struct event_trigger_data *data,
6836                                  struct trace_event_file *file)
6837 {
6838         struct hist_trigger_data *hist_data = data->private_data;
6839         struct event_trigger_data *test, *named_data = NULL;
6840         struct trace_array *tr = file->tr;
6841         int ret = 0;
6842
6843         if (hist_data->attrs->name) {
6844                 named_data = find_named_trigger(hist_data->attrs->name);
6845                 if (named_data) {
6846                         if (!hist_trigger_match(data, named_data, named_data,
6847                                                 true)) {
6848                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6849                                 ret = -EINVAL;
6850                                 goto out;
6851                         }
6852                 }
6853         }
6854
6855         if (hist_data->attrs->name && !named_data)
6856                 goto new;
6857
6858         lockdep_assert_held(&event_mutex);
6859
6860         list_for_each_entry(test, &file->triggers, list) {
6861                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6862                         if (!hist_trigger_match(data, test, named_data, false))
6863                                 continue;
6864                         if (hist_data->attrs->pause)
6865                                 test->paused = true;
6866                         else if (hist_data->attrs->cont)
6867                                 test->paused = false;
6868                         else if (hist_data->attrs->clear)
6869                                 hist_clear(test);
6870                         else {
6871                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6872                                 ret = -EEXIST;
6873                         }
6874                         goto out;
6875                 }
6876         }
6877  new:
6878         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6879                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6880                 ret = -ENOENT;
6881                 goto out;
6882         }
6883
6884         if (hist_data->attrs->pause)
6885                 data->paused = true;
6886
6887         if (named_data) {
6888                 data->private_data = named_data->private_data;
6889                 set_named_trigger_data(data, named_data);
6890                 data->ops = &event_hist_trigger_named_ops;
6891         }
6892
6893         if (data->ops->init) {
6894                 ret = data->ops->init(data->ops, data);
6895                 if (ret < 0)
6896                         goto out;
6897         }
6898
6899         if (hist_data->enable_timestamps) {
6900                 char *clock = hist_data->attrs->clock;
6901
6902                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6903                 if (ret) {
6904                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6905                         goto out;
6906                 }
6907
6908                 tracing_set_time_stamp_abs(file->tr, true);
6909         }
6910
6911         if (named_data)
6912                 destroy_hist_data(hist_data);
6913
6914         ret++;
6915  out:
6916         return ret;
6917 }
6918
6919 static int hist_trigger_enable(struct event_trigger_data *data,
6920                                struct trace_event_file *file)
6921 {
6922         int ret = 0;
6923
6924         list_add_tail_rcu(&data->list, &file->triggers);
6925
6926         update_cond_flag(file);
6927
6928         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6929                 list_del_rcu(&data->list);
6930                 update_cond_flag(file);
6931                 ret--;
6932         }
6933
6934         return ret;
6935 }
6936
6937 static bool have_hist_trigger_match(struct event_trigger_data *data,
6938                                     struct trace_event_file *file)
6939 {
6940         struct hist_trigger_data *hist_data = data->private_data;
6941         struct event_trigger_data *test, *named_data = NULL;
6942         bool match = false;
6943
6944         lockdep_assert_held(&event_mutex);
6945
6946         if (hist_data->attrs->name)
6947                 named_data = find_named_trigger(hist_data->attrs->name);
6948
6949         list_for_each_entry(test, &file->triggers, list) {
6950                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6951                         if (hist_trigger_match(data, test, named_data, false)) {
6952                                 match = true;
6953                                 break;
6954                         }
6955                 }
6956         }
6957
6958         return match;
6959 }
6960
6961 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6962                                     struct trace_event_file *file)
6963 {
6964         struct hist_trigger_data *hist_data = data->private_data;
6965         struct event_trigger_data *test, *named_data = NULL;
6966
6967         lockdep_assert_held(&event_mutex);
6968
6969         if (hist_data->attrs->name)
6970                 named_data = find_named_trigger(hist_data->attrs->name);
6971
6972         list_for_each_entry(test, &file->triggers, list) {
6973                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6974                         if (!hist_trigger_match(data, test, named_data, false))
6975                                 continue;
6976                         hist_data = test->private_data;
6977                         if (check_var_refs(hist_data))
6978                                 return true;
6979                         break;
6980                 }
6981         }
6982
6983         return false;
6984 }
6985
6986 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
6987                                     struct event_trigger_data *data,
6988                                     struct trace_event_file *file)
6989 {
6990         struct hist_trigger_data *hist_data = data->private_data;
6991         struct event_trigger_data *test, *named_data = NULL;
6992         bool unregistered = false;
6993
6994         lockdep_assert_held(&event_mutex);
6995
6996         if (hist_data->attrs->name)
6997                 named_data = find_named_trigger(hist_data->attrs->name);
6998
6999         list_for_each_entry(test, &file->triggers, list) {
7000                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7001                         if (!hist_trigger_match(data, test, named_data, false))
7002                                 continue;
7003                         unregistered = true;
7004                         list_del_rcu(&test->list);
7005                         trace_event_trigger_enable_disable(file, 0);
7006                         update_cond_flag(file);
7007                         break;
7008                 }
7009         }
7010
7011         if (unregistered && test->ops->free)
7012                 test->ops->free(test->ops, test);
7013
7014         if (hist_data->enable_timestamps) {
7015                 if (!hist_data->remove || unregistered)
7016                         tracing_set_time_stamp_abs(file->tr, false);
7017         }
7018 }
7019
7020 static bool hist_file_check_refs(struct trace_event_file *file)
7021 {
7022         struct hist_trigger_data *hist_data;
7023         struct event_trigger_data *test;
7024
7025         lockdep_assert_held(&event_mutex);
7026
7027         list_for_each_entry(test, &file->triggers, list) {
7028                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7029                         hist_data = test->private_data;
7030                         if (check_var_refs(hist_data))
7031                                 return true;
7032                 }
7033         }
7034
7035         return false;
7036 }
7037
7038 static void hist_unreg_all(struct trace_event_file *file)
7039 {
7040         struct event_trigger_data *test, *n;
7041         struct hist_trigger_data *hist_data;
7042         struct synth_event *se;
7043         const char *se_name;
7044
7045         lockdep_assert_held(&event_mutex);
7046
7047         if (hist_file_check_refs(file))
7048                 return;
7049
7050         list_for_each_entry_safe(test, n, &file->triggers, list) {
7051                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7052                         hist_data = test->private_data;
7053                         list_del_rcu(&test->list);
7054                         trace_event_trigger_enable_disable(file, 0);
7055
7056                         se_name = trace_event_name(file->event_call);
7057                         se = find_synth_event(se_name);
7058                         if (se)
7059                                 se->ref--;
7060
7061                         update_cond_flag(file);
7062                         if (hist_data->enable_timestamps)
7063                                 tracing_set_time_stamp_abs(file->tr, false);
7064                         if (test->ops->free)
7065                                 test->ops->free(test->ops, test);
7066                 }
7067         }
7068 }
7069
7070 static int event_hist_trigger_func(struct event_command *cmd_ops,
7071                                    struct trace_event_file *file,
7072                                    char *glob, char *cmd, char *param)
7073 {
7074         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
7075         struct event_trigger_data *trigger_data;
7076         struct hist_trigger_attrs *attrs;
7077         struct event_trigger_ops *trigger_ops;
7078         struct hist_trigger_data *hist_data;
7079         struct synth_event *se;
7080         const char *se_name;
7081         bool remove = false;
7082         char *trigger, *p;
7083         int ret = 0;
7084
7085         lockdep_assert_held(&event_mutex);
7086
7087         if (glob && strlen(glob)) {
7088                 hist_err_clear();
7089                 last_cmd_set(file, param);
7090         }
7091
7092         if (!param)
7093                 return -EINVAL;
7094
7095         if (glob[0] == '!')
7096                 remove = true;
7097
7098         /*
7099          * separate the trigger from the filter (k:v [if filter])
7100          * allowing for whitespace in the trigger
7101          */
7102         p = trigger = param;
7103         do {
7104                 p = strstr(p, "if");
7105                 if (!p)
7106                         break;
7107                 if (p == param)
7108                         return -EINVAL;
7109                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
7110                         p++;
7111                         continue;
7112                 }
7113                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
7114                         return -EINVAL;
7115                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
7116                         p++;
7117                         continue;
7118                 }
7119                 break;
7120         } while (p);
7121
7122         if (!p)
7123                 param = NULL;
7124         else {
7125                 *(p - 1) = '\0';
7126                 param = strstrip(p);
7127                 trigger = strstrip(trigger);
7128         }
7129
7130         attrs = parse_hist_trigger_attrs(file->tr, trigger);
7131         if (IS_ERR(attrs))
7132                 return PTR_ERR(attrs);
7133
7134         if (attrs->map_bits)
7135                 hist_trigger_bits = attrs->map_bits;
7136
7137         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7138         if (IS_ERR(hist_data)) {
7139                 destroy_hist_trigger_attrs(attrs);
7140                 return PTR_ERR(hist_data);
7141         }
7142
7143         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
7144
7145         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
7146         if (!trigger_data) {
7147                 ret = -ENOMEM;
7148                 goto out_free;
7149         }
7150
7151         trigger_data->count = -1;
7152         trigger_data->ops = trigger_ops;
7153         trigger_data->cmd_ops = cmd_ops;
7154
7155         INIT_LIST_HEAD(&trigger_data->list);
7156         RCU_INIT_POINTER(trigger_data->filter, NULL);
7157
7158         trigger_data->private_data = hist_data;
7159
7160         /* if param is non-empty, it's supposed to be a filter */
7161         if (param && cmd_ops->set_filter) {
7162                 ret = cmd_ops->set_filter(param, trigger_data, file);
7163                 if (ret < 0)
7164                         goto out_free;
7165         }
7166
7167         if (remove) {
7168                 if (!have_hist_trigger_match(trigger_data, file))
7169                         goto out_free;
7170
7171                 if (hist_trigger_check_refs(trigger_data, file)) {
7172                         ret = -EBUSY;
7173                         goto out_free;
7174                 }
7175
7176                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7177                 se_name = trace_event_name(file->event_call);
7178                 se = find_synth_event(se_name);
7179                 if (se)
7180                         se->ref--;
7181                 ret = 0;
7182                 goto out_free;
7183         }
7184
7185         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
7186         /*
7187          * The above returns on success the # of triggers registered,
7188          * but if it didn't register any it returns zero.  Consider no
7189          * triggers registered a failure too.
7190          */
7191         if (!ret) {
7192                 if (!(attrs->pause || attrs->cont || attrs->clear))
7193                         ret = -ENOENT;
7194                 goto out_free;
7195         } else if (ret < 0)
7196                 goto out_free;
7197
7198         if (get_named_trigger_data(trigger_data))
7199                 goto enable;
7200
7201         if (has_hist_vars(hist_data))
7202                 save_hist_vars(hist_data);
7203
7204         ret = create_actions(hist_data);
7205         if (ret)
7206                 goto out_unreg;
7207
7208         ret = tracing_map_init(hist_data->map);
7209         if (ret)
7210                 goto out_unreg;
7211 enable:
7212         ret = hist_trigger_enable(trigger_data, file);
7213         if (ret)
7214                 goto out_unreg;
7215
7216         se_name = trace_event_name(file->event_call);
7217         se = find_synth_event(se_name);
7218         if (se)
7219                 se->ref++;
7220         /* Just return zero, not the number of registered triggers */
7221         ret = 0;
7222  out:
7223         if (ret == 0)
7224                 hist_err_clear();
7225
7226         return ret;
7227  out_unreg:
7228         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7229  out_free:
7230         if (cmd_ops->set_filter)
7231                 cmd_ops->set_filter(NULL, trigger_data, NULL);
7232
7233         remove_hist_vars(hist_data);
7234
7235         kfree(trigger_data);
7236
7237         destroy_hist_data(hist_data);
7238         goto out;
7239 }
7240
7241 static struct event_command trigger_hist_cmd = {
7242         .name                   = "hist",
7243         .trigger_type           = ETT_EVENT_HIST,
7244         .flags                  = EVENT_CMD_FL_NEEDS_REC,
7245         .func                   = event_hist_trigger_func,
7246         .reg                    = hist_register_trigger,
7247         .unreg                  = hist_unregister_trigger,
7248         .unreg_all              = hist_unreg_all,
7249         .get_trigger_ops        = event_hist_get_trigger_ops,
7250         .set_filter             = set_trigger_filter,
7251 };
7252
7253 __init int register_trigger_hist_cmd(void)
7254 {
7255         int ret;
7256
7257         ret = register_event_command(&trigger_hist_cmd);
7258         WARN_ON(ret < 0);
7259
7260         return ret;
7261 }
7262
7263 static void
7264 hist_enable_trigger(struct event_trigger_data *data, void *rec,
7265                     struct ring_buffer_event *event)
7266 {
7267         struct enable_trigger_data *enable_data = data->private_data;
7268         struct event_trigger_data *test;
7269
7270         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
7271                                 lockdep_is_held(&event_mutex)) {
7272                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7273                         if (enable_data->enable)
7274                                 test->paused = false;
7275                         else
7276                                 test->paused = true;
7277                 }
7278         }
7279 }
7280
7281 static void
7282 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
7283                           struct ring_buffer_event *event)
7284 {
7285         if (!data->count)
7286                 return;
7287
7288         if (data->count != -1)
7289                 (data->count)--;
7290
7291         hist_enable_trigger(data, rec, event);
7292 }
7293
7294 static struct event_trigger_ops hist_enable_trigger_ops = {
7295         .func                   = hist_enable_trigger,
7296         .print                  = event_enable_trigger_print,
7297         .init                   = event_trigger_init,
7298         .free                   = event_enable_trigger_free,
7299 };
7300
7301 static struct event_trigger_ops hist_enable_count_trigger_ops = {
7302         .func                   = hist_enable_count_trigger,
7303         .print                  = event_enable_trigger_print,
7304         .init                   = event_trigger_init,
7305         .free                   = event_enable_trigger_free,
7306 };
7307
7308 static struct event_trigger_ops hist_disable_trigger_ops = {
7309         .func                   = hist_enable_trigger,
7310         .print                  = event_enable_trigger_print,
7311         .init                   = event_trigger_init,
7312         .free                   = event_enable_trigger_free,
7313 };
7314
7315 static struct event_trigger_ops hist_disable_count_trigger_ops = {
7316         .func                   = hist_enable_count_trigger,
7317         .print                  = event_enable_trigger_print,
7318         .init                   = event_trigger_init,
7319         .free                   = event_enable_trigger_free,
7320 };
7321
7322 static struct event_trigger_ops *
7323 hist_enable_get_trigger_ops(char *cmd, char *param)
7324 {
7325         struct event_trigger_ops *ops;
7326         bool enable;
7327
7328         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
7329
7330         if (enable)
7331                 ops = param ? &hist_enable_count_trigger_ops :
7332                         &hist_enable_trigger_ops;
7333         else
7334                 ops = param ? &hist_disable_count_trigger_ops :
7335                         &hist_disable_trigger_ops;
7336
7337         return ops;
7338 }
7339
7340 static void hist_enable_unreg_all(struct trace_event_file *file)
7341 {
7342         struct event_trigger_data *test, *n;
7343
7344         list_for_each_entry_safe(test, n, &file->triggers, list) {
7345                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
7346                         list_del_rcu(&test->list);
7347                         update_cond_flag(file);
7348                         trace_event_trigger_enable_disable(file, 0);
7349                         if (test->ops->free)
7350                                 test->ops->free(test->ops, test);
7351                 }
7352         }
7353 }
7354
7355 static struct event_command trigger_hist_enable_cmd = {
7356         .name                   = ENABLE_HIST_STR,
7357         .trigger_type           = ETT_HIST_ENABLE,
7358         .func                   = event_enable_trigger_func,
7359         .reg                    = event_enable_register_trigger,
7360         .unreg                  = event_enable_unregister_trigger,
7361         .unreg_all              = hist_enable_unreg_all,
7362         .get_trigger_ops        = hist_enable_get_trigger_ops,
7363         .set_filter             = set_trigger_filter,
7364 };
7365
7366 static struct event_command trigger_hist_disable_cmd = {
7367         .name                   = DISABLE_HIST_STR,
7368         .trigger_type           = ETT_HIST_ENABLE,
7369         .func                   = event_enable_trigger_func,
7370         .reg                    = event_enable_register_trigger,
7371         .unreg                  = event_enable_unregister_trigger,
7372         .unreg_all              = hist_enable_unreg_all,
7373         .get_trigger_ops        = hist_enable_get_trigger_ops,
7374         .set_filter             = set_trigger_filter,
7375 };
7376
7377 static __init void unregister_trigger_hist_enable_disable_cmds(void)
7378 {
7379         unregister_event_command(&trigger_hist_enable_cmd);
7380         unregister_event_command(&trigger_hist_disable_cmd);
7381 }
7382
7383 __init int register_trigger_hist_enable_disable_cmds(void)
7384 {
7385         int ret;
7386
7387         ret = register_event_command(&trigger_hist_enable_cmd);
7388         if (WARN_ON(ret < 0))
7389                 return ret;
7390         ret = register_event_command(&trigger_hist_disable_cmd);
7391         if (WARN_ON(ret < 0))
7392                 unregister_trigger_hist_enable_disable_cmds();
7393
7394         return ret;
7395 }
7396
7397 static __init int trace_events_hist_init(void)
7398 {
7399         struct dentry *entry = NULL;
7400         struct dentry *d_tracer;
7401         int err = 0;
7402
7403         err = dyn_event_register(&synth_event_ops);
7404         if (err) {
7405                 pr_warn("Could not register synth_event_ops\n");
7406                 return err;
7407         }
7408
7409         d_tracer = tracing_init_dentry();
7410         if (IS_ERR(d_tracer)) {
7411                 err = PTR_ERR(d_tracer);
7412                 goto err;
7413         }
7414
7415         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
7416                                     NULL, &synth_events_fops);
7417         if (!entry) {
7418                 err = -ENODEV;
7419                 goto err;
7420         }
7421
7422         return err;
7423  err:
7424         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
7425
7426         return err;
7427 }
7428
7429 fs_initcall(trace_events_hist_init);