2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 __disable_trace_on_warning = 1;
161 __setup("traceoff_on_warning=", stop_trace_on_warning);
163 static int __init boot_alloc_snapshot(char *str)
165 allocate_snapshot = true;
166 /* We also need the main ring buffer expanded */
167 ring_buffer_expanded = true;
170 __setup("alloc_snapshot", boot_alloc_snapshot);
173 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
174 static char *trace_boot_options __initdata;
176 static int __init set_trace_boot_options(char *str)
178 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
179 trace_boot_options = trace_boot_options_buf;
182 __setup("trace_options=", set_trace_boot_options);
184 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
185 static char *trace_boot_clock __initdata;
187 static int __init set_trace_boot_clock(char *str)
189 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
190 trace_boot_clock = trace_boot_clock_buf;
193 __setup("trace_clock=", set_trace_boot_clock);
196 unsigned long long ns2usecs(cycle_t nsec)
204 * The global_trace is the descriptor that holds the tracing
205 * buffers for the live tracing. For each CPU, it contains
206 * a link list of pages that will store trace entries. The
207 * page descriptor of the pages in the memory is used to hold
208 * the link list by linking the lru item in the page descriptor
209 * to each of the pages in the buffer per CPU.
211 * For each active CPU there is a data field that holds the
212 * pages for the buffer for that CPU. Each CPU has the same number
213 * of pages allocated for its buffer.
215 static struct trace_array global_trace;
217 LIST_HEAD(ftrace_trace_arrays);
219 int trace_array_get(struct trace_array *this_tr)
221 struct trace_array *tr;
224 mutex_lock(&trace_types_lock);
225 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
232 mutex_unlock(&trace_types_lock);
237 static void __trace_array_put(struct trace_array *this_tr)
239 WARN_ON(!this_tr->ref);
243 void trace_array_put(struct trace_array *this_tr)
245 mutex_lock(&trace_types_lock);
246 __trace_array_put(this_tr);
247 mutex_unlock(&trace_types_lock);
250 int filter_check_discard(struct ftrace_event_file *file, void *rec,
251 struct ring_buffer *buffer,
252 struct ring_buffer_event *event)
254 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
255 !filter_match_preds(file->filter, rec)) {
256 ring_buffer_discard_commit(buffer, event);
262 EXPORT_SYMBOL_GPL(filter_check_discard);
264 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
265 struct ring_buffer *buffer,
266 struct ring_buffer_event *event)
268 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
269 !filter_match_preds(call->filter, rec)) {
270 ring_buffer_discard_commit(buffer, event);
276 EXPORT_SYMBOL_GPL(call_filter_check_discard);
278 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
282 /* Early boot up does not have a buffer yet */
284 return trace_clock_local();
286 ts = ring_buffer_time_stamp(buf->buffer, cpu);
287 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
292 cycle_t ftrace_now(int cpu)
294 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
298 * tracing_is_enabled - Show if global_trace has been disabled
300 * Shows if the global trace has been enabled or not. It uses the
301 * mirror flag "buffer_disabled" to be used in fast paths such as for
302 * the irqsoff tracer. But it may be inaccurate due to races. If you
303 * need to know the accurate state, use tracing_is_on() which is a little
304 * slower, but accurate.
306 int tracing_is_enabled(void)
309 * For quick access (irqsoff uses this in fast path), just
310 * return the mirror variable of the state of the ring buffer.
311 * It's a little racy, but we don't really care.
314 return !global_trace.buffer_disabled;
318 * trace_buf_size is the size in bytes that is allocated
319 * for a buffer. Note, the number of bytes is always rounded
322 * This number is purposely set to a low number of 16384.
323 * If the dump on oops happens, it will be much appreciated
324 * to not have to wait for all that output. Anyway this can be
325 * boot time and run time configurable.
327 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
329 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
331 /* trace_types holds a link list of available tracers. */
332 static struct tracer *trace_types __read_mostly;
335 * trace_types_lock is used to protect the trace_types list.
337 DEFINE_MUTEX(trace_types_lock);
340 * serialize the access of the ring buffer
342 * ring buffer serializes readers, but it is low level protection.
343 * The validity of the events (which returns by ring_buffer_peek() ..etc)
344 * are not protected by ring buffer.
346 * The content of events may become garbage if we allow other process consumes
347 * these events concurrently:
348 * A) the page of the consumed events may become a normal page
349 * (not reader page) in ring buffer, and this page will be rewrited
350 * by events producer.
351 * B) The page of the consumed events may become a page for splice_read,
352 * and this page will be returned to system.
354 * These primitives allow multi process access to different cpu ring buffer
357 * These primitives don't distinguish read-only and read-consume access.
358 * Multi read-only access are also serialized.
362 static DECLARE_RWSEM(all_cpu_access_lock);
363 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
365 static inline void trace_access_lock(int cpu)
367 if (cpu == RING_BUFFER_ALL_CPUS) {
368 /* gain it for accessing the whole ring buffer. */
369 down_write(&all_cpu_access_lock);
371 /* gain it for accessing a cpu ring buffer. */
373 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
374 down_read(&all_cpu_access_lock);
376 /* Secondly block other access to this @cpu ring buffer. */
377 mutex_lock(&per_cpu(cpu_access_lock, cpu));
381 static inline void trace_access_unlock(int cpu)
383 if (cpu == RING_BUFFER_ALL_CPUS) {
384 up_write(&all_cpu_access_lock);
386 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
387 up_read(&all_cpu_access_lock);
391 static inline void trace_access_lock_init(void)
395 for_each_possible_cpu(cpu)
396 mutex_init(&per_cpu(cpu_access_lock, cpu));
401 static DEFINE_MUTEX(access_lock);
403 static inline void trace_access_lock(int cpu)
406 mutex_lock(&access_lock);
409 static inline void trace_access_unlock(int cpu)
412 mutex_unlock(&access_lock);
415 static inline void trace_access_lock_init(void)
421 /* trace_flags holds trace_options default values */
422 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
423 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
424 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
425 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
427 static void tracer_tracing_on(struct trace_array *tr)
429 if (tr->trace_buffer.buffer)
430 ring_buffer_record_on(tr->trace_buffer.buffer);
432 * This flag is looked at when buffers haven't been allocated
433 * yet, or by some tracers (like irqsoff), that just want to
434 * know if the ring buffer has been disabled, but it can handle
435 * races of where it gets disabled but we still do a record.
436 * As the check is in the fast path of the tracers, it is more
437 * important to be fast than accurate.
439 tr->buffer_disabled = 0;
440 /* Make the flag seen by readers */
445 * tracing_on - enable tracing buffers
447 * This function enables tracing buffers that may have been
448 * disabled with tracing_off.
450 void tracing_on(void)
452 tracer_tracing_on(&global_trace);
454 EXPORT_SYMBOL_GPL(tracing_on);
457 * __trace_puts - write a constant string into the trace buffer.
458 * @ip: The address of the caller
459 * @str: The constant string to write
460 * @size: The size of the string.
462 int __trace_puts(unsigned long ip, const char *str, int size)
464 struct ring_buffer_event *event;
465 struct ring_buffer *buffer;
466 struct print_entry *entry;
467 unsigned long irq_flags;
471 if (!(trace_flags & TRACE_ITER_PRINTK))
474 pc = preempt_count();
476 if (unlikely(tracing_selftest_running || tracing_disabled))
479 alloc = sizeof(*entry) + size + 2; /* possible \n added */
481 local_save_flags(irq_flags);
482 buffer = global_trace.trace_buffer.buffer;
483 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
488 entry = ring_buffer_event_data(event);
491 memcpy(&entry->buf, str, size);
493 /* Add a newline if necessary */
494 if (entry->buf[size - 1] != '\n') {
495 entry->buf[size] = '\n';
496 entry->buf[size + 1] = '\0';
498 entry->buf[size] = '\0';
500 __buffer_unlock_commit(buffer, event);
501 ftrace_trace_stack(buffer, irq_flags, 4, pc);
505 EXPORT_SYMBOL_GPL(__trace_puts);
508 * __trace_bputs - write the pointer to a constant string into trace buffer
509 * @ip: The address of the caller
510 * @str: The constant string to write to the buffer to
512 int __trace_bputs(unsigned long ip, const char *str)
514 struct ring_buffer_event *event;
515 struct ring_buffer *buffer;
516 struct bputs_entry *entry;
517 unsigned long irq_flags;
518 int size = sizeof(struct bputs_entry);
521 if (!(trace_flags & TRACE_ITER_PRINTK))
524 pc = preempt_count();
526 if (unlikely(tracing_selftest_running || tracing_disabled))
529 local_save_flags(irq_flags);
530 buffer = global_trace.trace_buffer.buffer;
531 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
536 entry = ring_buffer_event_data(event);
540 __buffer_unlock_commit(buffer, event);
541 ftrace_trace_stack(buffer, irq_flags, 4, pc);
545 EXPORT_SYMBOL_GPL(__trace_bputs);
547 #ifdef CONFIG_TRACER_SNAPSHOT
549 * trace_snapshot - take a snapshot of the current buffer.
551 * This causes a swap between the snapshot buffer and the current live
552 * tracing buffer. You can use this to take snapshots of the live
553 * trace when some condition is triggered, but continue to trace.
555 * Note, make sure to allocate the snapshot with either
556 * a tracing_snapshot_alloc(), or by doing it manually
557 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
559 * If the snapshot buffer is not allocated, it will stop tracing.
560 * Basically making a permanent snapshot.
562 void tracing_snapshot(void)
564 struct trace_array *tr = &global_trace;
565 struct tracer *tracer = tr->current_trace;
569 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
570 internal_trace_puts("*** snapshot is being ignored ***\n");
574 if (!tr->allocated_snapshot) {
575 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
576 internal_trace_puts("*** stopping trace here! ***\n");
581 /* Note, snapshot can not be used when the tracer uses it */
582 if (tracer->use_max_tr) {
583 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
584 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
588 local_irq_save(flags);
589 update_max_tr(tr, current, smp_processor_id());
590 local_irq_restore(flags);
592 EXPORT_SYMBOL_GPL(tracing_snapshot);
594 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
595 struct trace_buffer *size_buf, int cpu_id);
596 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
598 static int alloc_snapshot(struct trace_array *tr)
602 if (!tr->allocated_snapshot) {
604 /* allocate spare buffer */
605 ret = resize_buffer_duplicate_size(&tr->max_buffer,
606 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
610 tr->allocated_snapshot = true;
616 static void free_snapshot(struct trace_array *tr)
619 * We don't free the ring buffer. instead, resize it because
620 * The max_tr ring buffer has some state (e.g. ring->clock) and
621 * we want preserve it.
623 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
624 set_buffer_entries(&tr->max_buffer, 1);
625 tracing_reset_online_cpus(&tr->max_buffer);
626 tr->allocated_snapshot = false;
630 * tracing_alloc_snapshot - allocate snapshot buffer.
632 * This only allocates the snapshot buffer if it isn't already
633 * allocated - it doesn't also take a snapshot.
635 * This is meant to be used in cases where the snapshot buffer needs
636 * to be set up for events that can't sleep but need to be able to
637 * trigger a snapshot.
639 int tracing_alloc_snapshot(void)
641 struct trace_array *tr = &global_trace;
644 ret = alloc_snapshot(tr);
649 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
652 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
654 * This is similar to trace_snapshot(), but it will allocate the
655 * snapshot buffer if it isn't already allocated. Use this only
656 * where it is safe to sleep, as the allocation may sleep.
658 * This causes a swap between the snapshot buffer and the current live
659 * tracing buffer. You can use this to take snapshots of the live
660 * trace when some condition is triggered, but continue to trace.
662 void tracing_snapshot_alloc(void)
666 ret = tracing_alloc_snapshot();
672 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
674 void tracing_snapshot(void)
676 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
678 EXPORT_SYMBOL_GPL(tracing_snapshot);
679 int tracing_alloc_snapshot(void)
681 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
684 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
685 void tracing_snapshot_alloc(void)
690 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
691 #endif /* CONFIG_TRACER_SNAPSHOT */
693 static void tracer_tracing_off(struct trace_array *tr)
695 if (tr->trace_buffer.buffer)
696 ring_buffer_record_off(tr->trace_buffer.buffer);
698 * This flag is looked at when buffers haven't been allocated
699 * yet, or by some tracers (like irqsoff), that just want to
700 * know if the ring buffer has been disabled, but it can handle
701 * races of where it gets disabled but we still do a record.
702 * As the check is in the fast path of the tracers, it is more
703 * important to be fast than accurate.
705 tr->buffer_disabled = 1;
706 /* Make the flag seen by readers */
711 * tracing_off - turn off tracing buffers
713 * This function stops the tracing buffers from recording data.
714 * It does not disable any overhead the tracers themselves may
715 * be causing. This function simply causes all recording to
716 * the ring buffers to fail.
718 void tracing_off(void)
720 tracer_tracing_off(&global_trace);
722 EXPORT_SYMBOL_GPL(tracing_off);
724 void disable_trace_on_warning(void)
726 if (__disable_trace_on_warning)
731 * tracer_tracing_is_on - show real state of ring buffer enabled
732 * @tr : the trace array to know if ring buffer is enabled
734 * Shows real state of the ring buffer if it is enabled or not.
736 static int tracer_tracing_is_on(struct trace_array *tr)
738 if (tr->trace_buffer.buffer)
739 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
740 return !tr->buffer_disabled;
744 * tracing_is_on - show state of ring buffers enabled
746 int tracing_is_on(void)
748 return tracer_tracing_is_on(&global_trace);
750 EXPORT_SYMBOL_GPL(tracing_is_on);
752 static int __init set_buf_size(char *str)
754 unsigned long buf_size;
758 buf_size = memparse(str, &str);
759 /* nr_entries can not be zero */
762 trace_buf_size = buf_size;
765 __setup("trace_buf_size=", set_buf_size);
767 static int __init set_tracing_thresh(char *str)
769 unsigned long threshold;
774 ret = kstrtoul(str, 0, &threshold);
777 tracing_thresh = threshold * 1000;
780 __setup("tracing_thresh=", set_tracing_thresh);
782 unsigned long nsecs_to_usecs(unsigned long nsecs)
787 /* These must match the bit postions in trace_iterator_flags */
788 static const char *trace_options[] = {
821 int in_ns; /* is this clock in nanoseconds? */
823 { trace_clock_local, "local", 1 },
824 { trace_clock_global, "global", 1 },
825 { trace_clock_counter, "counter", 0 },
826 { trace_clock_jiffies, "uptime", 0 },
827 { trace_clock, "perf", 1 },
832 * trace_parser_get_init - gets the buffer for trace parser
834 int trace_parser_get_init(struct trace_parser *parser, int size)
836 memset(parser, 0, sizeof(*parser));
838 parser->buffer = kmalloc(size, GFP_KERNEL);
847 * trace_parser_put - frees the buffer for trace parser
849 void trace_parser_put(struct trace_parser *parser)
851 kfree(parser->buffer);
855 * trace_get_user - reads the user input string separated by space
856 * (matched by isspace(ch))
858 * For each string found the 'struct trace_parser' is updated,
859 * and the function returns.
861 * Returns number of bytes read.
863 * See kernel/trace/trace.h for 'struct trace_parser' details.
865 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
866 size_t cnt, loff_t *ppos)
873 trace_parser_clear(parser);
875 ret = get_user(ch, ubuf++);
883 * The parser is not finished with the last write,
884 * continue reading the user input without skipping spaces.
887 /* skip white space */
888 while (cnt && isspace(ch)) {
889 ret = get_user(ch, ubuf++);
896 /* only spaces were written */
906 /* read the non-space input */
907 while (cnt && !isspace(ch)) {
908 if (parser->idx < parser->size - 1)
909 parser->buffer[parser->idx++] = ch;
914 ret = get_user(ch, ubuf++);
921 /* We either got finished input or we have to wait for another call. */
923 parser->buffer[parser->idx] = 0;
924 parser->cont = false;
925 } else if (parser->idx < parser->size - 1) {
927 parser->buffer[parser->idx++] = ch;
940 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
948 if (s->len <= s->readpos)
951 len = s->len - s->readpos;
954 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
964 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
968 if (s->len <= s->readpos)
971 len = s->len - s->readpos;
974 memcpy(buf, s->buffer + s->readpos, cnt);
980 unsigned long __read_mostly tracing_thresh;
982 #ifdef CONFIG_TRACER_MAX_TRACE
984 * Copy the new maximum trace into the separate maximum-trace
985 * structure. (this way the maximum trace is permanently saved,
986 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
989 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
991 struct trace_buffer *trace_buf = &tr->trace_buffer;
992 struct trace_buffer *max_buf = &tr->max_buffer;
993 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
994 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
997 max_buf->time_start = data->preempt_timestamp;
999 max_data->saved_latency = tr->max_latency;
1000 max_data->critical_start = data->critical_start;
1001 max_data->critical_end = data->critical_end;
1003 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1004 max_data->pid = tsk->pid;
1006 * If tsk == current, then use current_uid(), as that does not use
1007 * RCU. The irq tracer can be called out of RCU scope.
1010 max_data->uid = current_uid();
1012 max_data->uid = task_uid(tsk);
1014 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1015 max_data->policy = tsk->policy;
1016 max_data->rt_priority = tsk->rt_priority;
1018 /* record this tasks comm */
1019 tracing_record_cmdline(tsk);
1023 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1025 * @tsk: the task with the latency
1026 * @cpu: The cpu that initiated the trace.
1028 * Flip the buffers between the @tr and the max_tr and record information
1029 * about which task was the cause of this latency.
1032 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1034 struct ring_buffer *buf;
1039 WARN_ON_ONCE(!irqs_disabled());
1041 if (!tr->allocated_snapshot) {
1042 /* Only the nop tracer should hit this when disabling */
1043 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1047 arch_spin_lock(&tr->max_lock);
1049 buf = tr->trace_buffer.buffer;
1050 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1051 tr->max_buffer.buffer = buf;
1053 __update_max_tr(tr, tsk, cpu);
1054 arch_spin_unlock(&tr->max_lock);
1058 * update_max_tr_single - only copy one trace over, and reset the rest
1060 * @tsk - task with the latency
1061 * @cpu - the cpu of the buffer to copy.
1063 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1066 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1073 WARN_ON_ONCE(!irqs_disabled());
1074 if (!tr->allocated_snapshot) {
1075 /* Only the nop tracer should hit this when disabling */
1076 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1080 arch_spin_lock(&tr->max_lock);
1082 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1084 if (ret == -EBUSY) {
1086 * We failed to swap the buffer due to a commit taking
1087 * place on this CPU. We fail to record, but we reset
1088 * the max trace buffer (no one writes directly to it)
1089 * and flag that it failed.
1091 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1092 "Failed to swap buffers due to commit in progress\n");
1095 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1097 __update_max_tr(tr, tsk, cpu);
1098 arch_spin_unlock(&tr->max_lock);
1100 #endif /* CONFIG_TRACER_MAX_TRACE */
1102 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1104 /* Iterators are static, they should be filled or empty */
1105 if (trace_buffer_iter(iter, iter->cpu_file))
1108 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1112 #ifdef CONFIG_FTRACE_STARTUP_TEST
1113 static int run_tracer_selftest(struct tracer *type)
1115 struct trace_array *tr = &global_trace;
1116 struct tracer *saved_tracer = tr->current_trace;
1119 if (!type->selftest || tracing_selftest_disabled)
1123 * Run a selftest on this tracer.
1124 * Here we reset the trace buffer, and set the current
1125 * tracer to be this tracer. The tracer can then run some
1126 * internal tracing to verify that everything is in order.
1127 * If we fail, we do not register this tracer.
1129 tracing_reset_online_cpus(&tr->trace_buffer);
1131 tr->current_trace = type;
1133 #ifdef CONFIG_TRACER_MAX_TRACE
1134 if (type->use_max_tr) {
1135 /* If we expanded the buffers, make sure the max is expanded too */
1136 if (ring_buffer_expanded)
1137 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1138 RING_BUFFER_ALL_CPUS);
1139 tr->allocated_snapshot = true;
1143 /* the test is responsible for initializing and enabling */
1144 pr_info("Testing tracer %s: ", type->name);
1145 ret = type->selftest(type, tr);
1146 /* the test is responsible for resetting too */
1147 tr->current_trace = saved_tracer;
1149 printk(KERN_CONT "FAILED!\n");
1150 /* Add the warning after printing 'FAILED' */
1154 /* Only reset on passing, to avoid touching corrupted buffers */
1155 tracing_reset_online_cpus(&tr->trace_buffer);
1157 #ifdef CONFIG_TRACER_MAX_TRACE
1158 if (type->use_max_tr) {
1159 tr->allocated_snapshot = false;
1161 /* Shrink the max buffer again */
1162 if (ring_buffer_expanded)
1163 ring_buffer_resize(tr->max_buffer.buffer, 1,
1164 RING_BUFFER_ALL_CPUS);
1168 printk(KERN_CONT "PASSED\n");
1172 static inline int run_tracer_selftest(struct tracer *type)
1176 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1179 * register_tracer - register a tracer with the ftrace system.
1180 * @type - the plugin for the tracer
1182 * Register a new plugin tracer.
1184 int register_tracer(struct tracer *type)
1190 pr_info("Tracer must have a name\n");
1194 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1195 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1199 mutex_lock(&trace_types_lock);
1201 tracing_selftest_running = true;
1203 for (t = trace_types; t; t = t->next) {
1204 if (strcmp(type->name, t->name) == 0) {
1206 pr_info("Tracer %s already registered\n",
1213 if (!type->set_flag)
1214 type->set_flag = &dummy_set_flag;
1216 type->flags = &dummy_tracer_flags;
1218 if (!type->flags->opts)
1219 type->flags->opts = dummy_tracer_opt;
1221 ret = run_tracer_selftest(type);
1225 type->next = trace_types;
1229 tracing_selftest_running = false;
1230 mutex_unlock(&trace_types_lock);
1232 if (ret || !default_bootup_tracer)
1235 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1238 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1239 /* Do we want this tracer to start on bootup? */
1240 tracing_set_tracer(&global_trace, type->name);
1241 default_bootup_tracer = NULL;
1242 /* disable other selftests, since this will break it. */
1243 tracing_selftest_disabled = true;
1244 #ifdef CONFIG_FTRACE_STARTUP_TEST
1245 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1253 void tracing_reset(struct trace_buffer *buf, int cpu)
1255 struct ring_buffer *buffer = buf->buffer;
1260 ring_buffer_record_disable(buffer);
1262 /* Make sure all commits have finished */
1263 synchronize_sched();
1264 ring_buffer_reset_cpu(buffer, cpu);
1266 ring_buffer_record_enable(buffer);
1269 void tracing_reset_online_cpus(struct trace_buffer *buf)
1271 struct ring_buffer *buffer = buf->buffer;
1277 ring_buffer_record_disable(buffer);
1279 /* Make sure all commits have finished */
1280 synchronize_sched();
1282 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1284 for_each_online_cpu(cpu)
1285 ring_buffer_reset_cpu(buffer, cpu);
1287 ring_buffer_record_enable(buffer);
1290 /* Must have trace_types_lock held */
1291 void tracing_reset_all_online_cpus(void)
1293 struct trace_array *tr;
1295 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1296 tracing_reset_online_cpus(&tr->trace_buffer);
1297 #ifdef CONFIG_TRACER_MAX_TRACE
1298 tracing_reset_online_cpus(&tr->max_buffer);
1303 #define SAVED_CMDLINES_DEFAULT 128
1304 #define NO_CMDLINE_MAP UINT_MAX
1305 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1306 struct saved_cmdlines_buffer {
1307 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1308 unsigned *map_cmdline_to_pid;
1309 unsigned cmdline_num;
1311 char *saved_cmdlines;
1313 static struct saved_cmdlines_buffer *savedcmd;
1315 /* temporary disable recording */
1316 static atomic_t trace_record_cmdline_disabled __read_mostly;
1318 static inline char *get_saved_cmdlines(int idx)
1320 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1323 static inline void set_cmdline(int idx, const char *cmdline)
1325 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1328 static int allocate_cmdlines_buffer(unsigned int val,
1329 struct saved_cmdlines_buffer *s)
1331 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1333 if (!s->map_cmdline_to_pid)
1336 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1337 if (!s->saved_cmdlines) {
1338 kfree(s->map_cmdline_to_pid);
1343 s->cmdline_num = val;
1344 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1345 sizeof(s->map_pid_to_cmdline));
1346 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1347 val * sizeof(*s->map_cmdline_to_pid));
1352 static int trace_create_savedcmd(void)
1356 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1360 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1370 int is_tracing_stopped(void)
1372 return global_trace.stop_count;
1376 * tracing_start - quick start of the tracer
1378 * If tracing is enabled but was stopped by tracing_stop,
1379 * this will start the tracer back up.
1381 void tracing_start(void)
1383 struct ring_buffer *buffer;
1384 unsigned long flags;
1386 if (tracing_disabled)
1389 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1390 if (--global_trace.stop_count) {
1391 if (global_trace.stop_count < 0) {
1392 /* Someone screwed up their debugging */
1394 global_trace.stop_count = 0;
1399 /* Prevent the buffers from switching */
1400 arch_spin_lock(&global_trace.max_lock);
1402 buffer = global_trace.trace_buffer.buffer;
1404 ring_buffer_record_enable(buffer);
1406 #ifdef CONFIG_TRACER_MAX_TRACE
1407 buffer = global_trace.max_buffer.buffer;
1409 ring_buffer_record_enable(buffer);
1412 arch_spin_unlock(&global_trace.max_lock);
1415 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1418 static void tracing_start_tr(struct trace_array *tr)
1420 struct ring_buffer *buffer;
1421 unsigned long flags;
1423 if (tracing_disabled)
1426 /* If global, we need to also start the max tracer */
1427 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1428 return tracing_start();
1430 raw_spin_lock_irqsave(&tr->start_lock, flags);
1432 if (--tr->stop_count) {
1433 if (tr->stop_count < 0) {
1434 /* Someone screwed up their debugging */
1441 buffer = tr->trace_buffer.buffer;
1443 ring_buffer_record_enable(buffer);
1446 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1450 * tracing_stop - quick stop of the tracer
1452 * Light weight way to stop tracing. Use in conjunction with
1455 void tracing_stop(void)
1457 struct ring_buffer *buffer;
1458 unsigned long flags;
1460 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1461 if (global_trace.stop_count++)
1464 /* Prevent the buffers from switching */
1465 arch_spin_lock(&global_trace.max_lock);
1467 buffer = global_trace.trace_buffer.buffer;
1469 ring_buffer_record_disable(buffer);
1471 #ifdef CONFIG_TRACER_MAX_TRACE
1472 buffer = global_trace.max_buffer.buffer;
1474 ring_buffer_record_disable(buffer);
1477 arch_spin_unlock(&global_trace.max_lock);
1480 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1483 static void tracing_stop_tr(struct trace_array *tr)
1485 struct ring_buffer *buffer;
1486 unsigned long flags;
1488 /* If global, we need to also stop the max tracer */
1489 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1490 return tracing_stop();
1492 raw_spin_lock_irqsave(&tr->start_lock, flags);
1493 if (tr->stop_count++)
1496 buffer = tr->trace_buffer.buffer;
1498 ring_buffer_record_disable(buffer);
1501 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1504 void trace_stop_cmdline_recording(void);
1506 static int trace_save_cmdline(struct task_struct *tsk)
1510 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1514 * It's not the end of the world if we don't get
1515 * the lock, but we also don't want to spin
1516 * nor do we want to disable interrupts,
1517 * so if we miss here, then better luck next time.
1519 if (!arch_spin_trylock(&trace_cmdline_lock))
1522 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1523 if (idx == NO_CMDLINE_MAP) {
1524 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1527 * Check whether the cmdline buffer at idx has a pid
1528 * mapped. We are going to overwrite that entry so we
1529 * need to clear the map_pid_to_cmdline. Otherwise we
1530 * would read the new comm for the old pid.
1532 pid = savedcmd->map_cmdline_to_pid[idx];
1533 if (pid != NO_CMDLINE_MAP)
1534 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1536 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1537 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1539 savedcmd->cmdline_idx = idx;
1542 set_cmdline(idx, tsk->comm);
1544 arch_spin_unlock(&trace_cmdline_lock);
1549 static void __trace_find_cmdline(int pid, char comm[])
1554 strcpy(comm, "<idle>");
1558 if (WARN_ON_ONCE(pid < 0)) {
1559 strcpy(comm, "<XXX>");
1563 if (pid > PID_MAX_DEFAULT) {
1564 strcpy(comm, "<...>");
1568 map = savedcmd->map_pid_to_cmdline[pid];
1569 if (map != NO_CMDLINE_MAP)
1570 strcpy(comm, get_saved_cmdlines(map));
1572 strcpy(comm, "<...>");
1575 void trace_find_cmdline(int pid, char comm[])
1578 arch_spin_lock(&trace_cmdline_lock);
1580 __trace_find_cmdline(pid, comm);
1582 arch_spin_unlock(&trace_cmdline_lock);
1586 void tracing_record_cmdline(struct task_struct *tsk)
1588 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1591 if (!__this_cpu_read(trace_cmdline_save))
1594 if (trace_save_cmdline(tsk))
1595 __this_cpu_write(trace_cmdline_save, false);
1599 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1602 struct task_struct *tsk = current;
1604 entry->preempt_count = pc & 0xff;
1605 entry->pid = (tsk) ? tsk->pid : 0;
1607 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1608 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1610 TRACE_FLAG_IRQS_NOSUPPORT |
1612 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1613 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1614 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1615 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1617 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1619 struct ring_buffer_event *
1620 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1623 unsigned long flags, int pc)
1625 struct ring_buffer_event *event;
1627 event = ring_buffer_lock_reserve(buffer, len);
1628 if (event != NULL) {
1629 struct trace_entry *ent = ring_buffer_event_data(event);
1631 tracing_generic_entry_update(ent, flags, pc);
1639 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1641 __this_cpu_write(trace_cmdline_save, true);
1642 ring_buffer_unlock_commit(buffer, event);
1646 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1647 struct ring_buffer_event *event,
1648 unsigned long flags, int pc)
1650 __buffer_unlock_commit(buffer, event);
1652 ftrace_trace_stack(buffer, flags, 6, pc);
1653 ftrace_trace_userstack(buffer, flags, pc);
1656 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1657 struct ring_buffer_event *event,
1658 unsigned long flags, int pc)
1660 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1662 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1664 static struct ring_buffer *temp_buffer;
1666 struct ring_buffer_event *
1667 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1668 struct ftrace_event_file *ftrace_file,
1669 int type, unsigned long len,
1670 unsigned long flags, int pc)
1672 struct ring_buffer_event *entry;
1674 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1675 entry = trace_buffer_lock_reserve(*current_rb,
1676 type, len, flags, pc);
1678 * If tracing is off, but we have triggers enabled
1679 * we still need to look at the event data. Use the temp_buffer
1680 * to store the trace event for the tigger to use. It's recusive
1681 * safe and will not be recorded anywhere.
1683 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1684 *current_rb = temp_buffer;
1685 entry = trace_buffer_lock_reserve(*current_rb,
1686 type, len, flags, pc);
1690 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1692 struct ring_buffer_event *
1693 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1694 int type, unsigned long len,
1695 unsigned long flags, int pc)
1697 *current_rb = global_trace.trace_buffer.buffer;
1698 return trace_buffer_lock_reserve(*current_rb,
1699 type, len, flags, pc);
1701 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1703 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1704 struct ring_buffer_event *event,
1705 unsigned long flags, int pc)
1707 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1709 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1711 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1712 struct ring_buffer_event *event,
1713 unsigned long flags, int pc,
1714 struct pt_regs *regs)
1716 __buffer_unlock_commit(buffer, event);
1718 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1719 ftrace_trace_userstack(buffer, flags, pc);
1721 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1723 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1724 struct ring_buffer_event *event)
1726 ring_buffer_discard_commit(buffer, event);
1728 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1731 trace_function(struct trace_array *tr,
1732 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1735 struct ftrace_event_call *call = &event_function;
1736 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1737 struct ring_buffer_event *event;
1738 struct ftrace_entry *entry;
1740 /* If we are reading the ring buffer, don't trace */
1741 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1744 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1748 entry = ring_buffer_event_data(event);
1750 entry->parent_ip = parent_ip;
1752 if (!call_filter_check_discard(call, entry, buffer, event))
1753 __buffer_unlock_commit(buffer, event);
1756 #ifdef CONFIG_STACKTRACE
1758 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1759 struct ftrace_stack {
1760 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1763 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1764 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1766 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1767 unsigned long flags,
1768 int skip, int pc, struct pt_regs *regs)
1770 struct ftrace_event_call *call = &event_kernel_stack;
1771 struct ring_buffer_event *event;
1772 struct stack_entry *entry;
1773 struct stack_trace trace;
1775 int size = FTRACE_STACK_ENTRIES;
1777 trace.nr_entries = 0;
1781 * Since events can happen in NMIs there's no safe way to
1782 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1783 * or NMI comes in, it will just have to use the default
1784 * FTRACE_STACK_SIZE.
1786 preempt_disable_notrace();
1788 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1790 * We don't need any atomic variables, just a barrier.
1791 * If an interrupt comes in, we don't care, because it would
1792 * have exited and put the counter back to what we want.
1793 * We just need a barrier to keep gcc from moving things
1797 if (use_stack == 1) {
1798 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1799 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1802 save_stack_trace_regs(regs, &trace);
1804 save_stack_trace(&trace);
1806 if (trace.nr_entries > size)
1807 size = trace.nr_entries;
1809 /* From now on, use_stack is a boolean */
1812 size *= sizeof(unsigned long);
1814 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1815 sizeof(*entry) + size, flags, pc);
1818 entry = ring_buffer_event_data(event);
1820 memset(&entry->caller, 0, size);
1823 memcpy(&entry->caller, trace.entries,
1824 trace.nr_entries * sizeof(unsigned long));
1826 trace.max_entries = FTRACE_STACK_ENTRIES;
1827 trace.entries = entry->caller;
1829 save_stack_trace_regs(regs, &trace);
1831 save_stack_trace(&trace);
1834 entry->size = trace.nr_entries;
1836 if (!call_filter_check_discard(call, entry, buffer, event))
1837 __buffer_unlock_commit(buffer, event);
1840 /* Again, don't let gcc optimize things here */
1842 __this_cpu_dec(ftrace_stack_reserve);
1843 preempt_enable_notrace();
1847 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1848 int skip, int pc, struct pt_regs *regs)
1850 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1853 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1856 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1859 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1862 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1865 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1868 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1872 * trace_dump_stack - record a stack back trace in the trace buffer
1873 * @skip: Number of functions to skip (helper handlers)
1875 void trace_dump_stack(int skip)
1877 unsigned long flags;
1879 if (tracing_disabled || tracing_selftest_running)
1882 local_save_flags(flags);
1885 * Skip 3 more, seems to get us at the caller of
1889 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1890 flags, skip, preempt_count(), NULL);
1893 static DEFINE_PER_CPU(int, user_stack_count);
1896 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1898 struct ftrace_event_call *call = &event_user_stack;
1899 struct ring_buffer_event *event;
1900 struct userstack_entry *entry;
1901 struct stack_trace trace;
1903 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1907 * NMIs can not handle page faults, even with fix ups.
1908 * The save user stack can (and often does) fault.
1910 if (unlikely(in_nmi()))
1914 * prevent recursion, since the user stack tracing may
1915 * trigger other kernel events.
1918 if (__this_cpu_read(user_stack_count))
1921 __this_cpu_inc(user_stack_count);
1923 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1924 sizeof(*entry), flags, pc);
1926 goto out_drop_count;
1927 entry = ring_buffer_event_data(event);
1929 entry->tgid = current->tgid;
1930 memset(&entry->caller, 0, sizeof(entry->caller));
1932 trace.nr_entries = 0;
1933 trace.max_entries = FTRACE_STACK_ENTRIES;
1935 trace.entries = entry->caller;
1937 save_stack_trace_user(&trace);
1938 if (!call_filter_check_discard(call, entry, buffer, event))
1939 __buffer_unlock_commit(buffer, event);
1942 __this_cpu_dec(user_stack_count);
1948 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1950 ftrace_trace_userstack(tr, flags, preempt_count());
1954 #endif /* CONFIG_STACKTRACE */
1956 /* created for use with alloc_percpu */
1957 struct trace_buffer_struct {
1958 char buffer[TRACE_BUF_SIZE];
1961 static struct trace_buffer_struct *trace_percpu_buffer;
1962 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1963 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1964 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1967 * The buffer used is dependent on the context. There is a per cpu
1968 * buffer for normal context, softirq contex, hard irq context and
1969 * for NMI context. Thise allows for lockless recording.
1971 * Note, if the buffers failed to be allocated, then this returns NULL
1973 static char *get_trace_buf(void)
1975 struct trace_buffer_struct *percpu_buffer;
1978 * If we have allocated per cpu buffers, then we do not
1979 * need to do any locking.
1982 percpu_buffer = trace_percpu_nmi_buffer;
1984 percpu_buffer = trace_percpu_irq_buffer;
1985 else if (in_softirq())
1986 percpu_buffer = trace_percpu_sirq_buffer;
1988 percpu_buffer = trace_percpu_buffer;
1993 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1996 static int alloc_percpu_trace_buffer(void)
1998 struct trace_buffer_struct *buffers;
1999 struct trace_buffer_struct *sirq_buffers;
2000 struct trace_buffer_struct *irq_buffers;
2001 struct trace_buffer_struct *nmi_buffers;
2003 buffers = alloc_percpu(struct trace_buffer_struct);
2007 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2011 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2015 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2019 trace_percpu_buffer = buffers;
2020 trace_percpu_sirq_buffer = sirq_buffers;
2021 trace_percpu_irq_buffer = irq_buffers;
2022 trace_percpu_nmi_buffer = nmi_buffers;
2027 free_percpu(irq_buffers);
2029 free_percpu(sirq_buffers);
2031 free_percpu(buffers);
2033 WARN(1, "Could not allocate percpu trace_printk buffer");
2037 static int buffers_allocated;
2039 void trace_printk_init_buffers(void)
2041 if (buffers_allocated)
2044 if (alloc_percpu_trace_buffer())
2047 /* trace_printk() is for debug use only. Don't use it in production. */
2049 pr_warning("\n**********************************************************\n");
2050 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2051 pr_warning("** **\n");
2052 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2053 pr_warning("** **\n");
2054 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2055 pr_warning("** unsafe for produciton use. **\n");
2056 pr_warning("** **\n");
2057 pr_warning("** If you see this message and you are not debugging **\n");
2058 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2059 pr_warning("** **\n");
2060 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2061 pr_warning("**********************************************************\n");
2063 /* Expand the buffers to set size */
2064 tracing_update_buffers();
2066 buffers_allocated = 1;
2069 * trace_printk_init_buffers() can be called by modules.
2070 * If that happens, then we need to start cmdline recording
2071 * directly here. If the global_trace.buffer is already
2072 * allocated here, then this was called by module code.
2074 if (global_trace.trace_buffer.buffer)
2075 tracing_start_cmdline_record();
2078 void trace_printk_start_comm(void)
2080 /* Start tracing comms if trace printk is set */
2081 if (!buffers_allocated)
2083 tracing_start_cmdline_record();
2086 static void trace_printk_start_stop_comm(int enabled)
2088 if (!buffers_allocated)
2092 tracing_start_cmdline_record();
2094 tracing_stop_cmdline_record();
2098 * trace_vbprintk - write binary msg to tracing buffer
2101 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2103 struct ftrace_event_call *call = &event_bprint;
2104 struct ring_buffer_event *event;
2105 struct ring_buffer *buffer;
2106 struct trace_array *tr = &global_trace;
2107 struct bprint_entry *entry;
2108 unsigned long flags;
2110 int len = 0, size, pc;
2112 if (unlikely(tracing_selftest_running || tracing_disabled))
2115 /* Don't pollute graph traces with trace_vprintk internals */
2116 pause_graph_tracing();
2118 pc = preempt_count();
2119 preempt_disable_notrace();
2121 tbuffer = get_trace_buf();
2127 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2129 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2132 local_save_flags(flags);
2133 size = sizeof(*entry) + sizeof(u32) * len;
2134 buffer = tr->trace_buffer.buffer;
2135 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2139 entry = ring_buffer_event_data(event);
2143 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2144 if (!call_filter_check_discard(call, entry, buffer, event)) {
2145 __buffer_unlock_commit(buffer, event);
2146 ftrace_trace_stack(buffer, flags, 6, pc);
2150 preempt_enable_notrace();
2151 unpause_graph_tracing();
2155 EXPORT_SYMBOL_GPL(trace_vbprintk);
2158 __trace_array_vprintk(struct ring_buffer *buffer,
2159 unsigned long ip, const char *fmt, va_list args)
2161 struct ftrace_event_call *call = &event_print;
2162 struct ring_buffer_event *event;
2163 int len = 0, size, pc;
2164 struct print_entry *entry;
2165 unsigned long flags;
2168 if (tracing_disabled || tracing_selftest_running)
2171 /* Don't pollute graph traces with trace_vprintk internals */
2172 pause_graph_tracing();
2174 pc = preempt_count();
2175 preempt_disable_notrace();
2178 tbuffer = get_trace_buf();
2184 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2185 if (len > TRACE_BUF_SIZE)
2188 local_save_flags(flags);
2189 size = sizeof(*entry) + len + 1;
2190 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2194 entry = ring_buffer_event_data(event);
2197 memcpy(&entry->buf, tbuffer, len);
2198 entry->buf[len] = '\0';
2199 if (!call_filter_check_discard(call, entry, buffer, event)) {
2200 __buffer_unlock_commit(buffer, event);
2201 ftrace_trace_stack(buffer, flags, 6, pc);
2204 preempt_enable_notrace();
2205 unpause_graph_tracing();
2210 int trace_array_vprintk(struct trace_array *tr,
2211 unsigned long ip, const char *fmt, va_list args)
2213 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2216 int trace_array_printk(struct trace_array *tr,
2217 unsigned long ip, const char *fmt, ...)
2222 if (!(trace_flags & TRACE_ITER_PRINTK))
2226 ret = trace_array_vprintk(tr, ip, fmt, ap);
2231 int trace_array_printk_buf(struct ring_buffer *buffer,
2232 unsigned long ip, const char *fmt, ...)
2237 if (!(trace_flags & TRACE_ITER_PRINTK))
2241 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2246 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2248 return trace_array_vprintk(&global_trace, ip, fmt, args);
2250 EXPORT_SYMBOL_GPL(trace_vprintk);
2252 static void trace_iterator_increment(struct trace_iterator *iter)
2254 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2258 ring_buffer_read(buf_iter, NULL);
2261 static struct trace_entry *
2262 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2263 unsigned long *lost_events)
2265 struct ring_buffer_event *event;
2266 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2269 event = ring_buffer_iter_peek(buf_iter, ts);
2271 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2275 iter->ent_size = ring_buffer_event_length(event);
2276 return ring_buffer_event_data(event);
2282 static struct trace_entry *
2283 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2284 unsigned long *missing_events, u64 *ent_ts)
2286 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2287 struct trace_entry *ent, *next = NULL;
2288 unsigned long lost_events = 0, next_lost = 0;
2289 int cpu_file = iter->cpu_file;
2290 u64 next_ts = 0, ts;
2296 * If we are in a per_cpu trace file, don't bother by iterating over
2297 * all cpu and peek directly.
2299 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2300 if (ring_buffer_empty_cpu(buffer, cpu_file))
2302 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2304 *ent_cpu = cpu_file;
2309 for_each_tracing_cpu(cpu) {
2311 if (ring_buffer_empty_cpu(buffer, cpu))
2314 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2317 * Pick the entry with the smallest timestamp:
2319 if (ent && (!next || ts < next_ts)) {
2323 next_lost = lost_events;
2324 next_size = iter->ent_size;
2328 iter->ent_size = next_size;
2331 *ent_cpu = next_cpu;
2337 *missing_events = next_lost;
2342 /* Find the next real entry, without updating the iterator itself */
2343 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2344 int *ent_cpu, u64 *ent_ts)
2346 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2349 /* Find the next real entry, and increment the iterator to the next entry */
2350 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2352 iter->ent = __find_next_entry(iter, &iter->cpu,
2353 &iter->lost_events, &iter->ts);
2356 trace_iterator_increment(iter);
2358 return iter->ent ? iter : NULL;
2361 static void trace_consume(struct trace_iterator *iter)
2363 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2364 &iter->lost_events);
2367 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2369 struct trace_iterator *iter = m->private;
2373 WARN_ON_ONCE(iter->leftover);
2377 /* can't go backwards */
2382 ent = trace_find_next_entry_inc(iter);
2386 while (ent && iter->idx < i)
2387 ent = trace_find_next_entry_inc(iter);
2394 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2396 struct ring_buffer_event *event;
2397 struct ring_buffer_iter *buf_iter;
2398 unsigned long entries = 0;
2401 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2403 buf_iter = trace_buffer_iter(iter, cpu);
2407 ring_buffer_iter_reset(buf_iter);
2410 * We could have the case with the max latency tracers
2411 * that a reset never took place on a cpu. This is evident
2412 * by the timestamp being before the start of the buffer.
2414 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2415 if (ts >= iter->trace_buffer->time_start)
2418 ring_buffer_read(buf_iter, NULL);
2421 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2425 * The current tracer is copied to avoid a global locking
2428 static void *s_start(struct seq_file *m, loff_t *pos)
2430 struct trace_iterator *iter = m->private;
2431 struct trace_array *tr = iter->tr;
2432 int cpu_file = iter->cpu_file;
2438 * copy the tracer to avoid using a global lock all around.
2439 * iter->trace is a copy of current_trace, the pointer to the
2440 * name may be used instead of a strcmp(), as iter->trace->name
2441 * will point to the same string as current_trace->name.
2443 mutex_lock(&trace_types_lock);
2444 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2445 *iter->trace = *tr->current_trace;
2446 mutex_unlock(&trace_types_lock);
2448 #ifdef CONFIG_TRACER_MAX_TRACE
2449 if (iter->snapshot && iter->trace->use_max_tr)
2450 return ERR_PTR(-EBUSY);
2453 if (!iter->snapshot)
2454 atomic_inc(&trace_record_cmdline_disabled);
2456 if (*pos != iter->pos) {
2461 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2462 for_each_tracing_cpu(cpu)
2463 tracing_iter_reset(iter, cpu);
2465 tracing_iter_reset(iter, cpu_file);
2468 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2473 * If we overflowed the seq_file before, then we want
2474 * to just reuse the trace_seq buffer again.
2480 p = s_next(m, p, &l);
2484 trace_event_read_lock();
2485 trace_access_lock(cpu_file);
2489 static void s_stop(struct seq_file *m, void *p)
2491 struct trace_iterator *iter = m->private;
2493 #ifdef CONFIG_TRACER_MAX_TRACE
2494 if (iter->snapshot && iter->trace->use_max_tr)
2498 if (!iter->snapshot)
2499 atomic_dec(&trace_record_cmdline_disabled);
2501 trace_access_unlock(iter->cpu_file);
2502 trace_event_read_unlock();
2506 get_total_entries(struct trace_buffer *buf,
2507 unsigned long *total, unsigned long *entries)
2509 unsigned long count;
2515 for_each_tracing_cpu(cpu) {
2516 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2518 * If this buffer has skipped entries, then we hold all
2519 * entries for the trace and we need to ignore the
2520 * ones before the time stamp.
2522 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2523 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2524 /* total is the same as the entries */
2528 ring_buffer_overrun_cpu(buf->buffer, cpu);
2533 static void print_lat_help_header(struct seq_file *m)
2535 seq_puts(m, "# _------=> CPU# \n");
2536 seq_puts(m, "# / _-----=> irqs-off \n");
2537 seq_puts(m, "# | / _----=> need-resched \n");
2538 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2539 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2540 seq_puts(m, "# |||| / delay \n");
2541 seq_puts(m, "# cmd pid ||||| time | caller \n");
2542 seq_puts(m, "# \\ / ||||| \\ | / \n");
2545 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2547 unsigned long total;
2548 unsigned long entries;
2550 get_total_entries(buf, &total, &entries);
2551 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2552 entries, total, num_online_cpus());
2556 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2558 print_event_info(buf, m);
2559 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2560 seq_puts(m, "# | | | | |\n");
2563 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2565 print_event_info(buf, m);
2566 seq_puts(m, "# _-----=> irqs-off\n");
2567 seq_puts(m, "# / _----=> need-resched\n");
2568 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2569 seq_puts(m, "# || / _--=> preempt-depth\n");
2570 seq_puts(m, "# ||| / delay\n");
2571 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2572 seq_puts(m, "# | | | |||| | |\n");
2576 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2578 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2579 struct trace_buffer *buf = iter->trace_buffer;
2580 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2581 struct tracer *type = iter->trace;
2582 unsigned long entries;
2583 unsigned long total;
2584 const char *name = "preemption";
2588 get_total_entries(buf, &total, &entries);
2590 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2592 seq_puts(m, "# -----------------------------------"
2593 "---------------------------------\n");
2594 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2595 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2596 nsecs_to_usecs(data->saved_latency),
2600 #if defined(CONFIG_PREEMPT_NONE)
2602 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2604 #elif defined(CONFIG_PREEMPT)
2609 /* These are reserved for later use */
2612 seq_printf(m, " #P:%d)\n", num_online_cpus());
2616 seq_puts(m, "# -----------------\n");
2617 seq_printf(m, "# | task: %.16s-%d "
2618 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2619 data->comm, data->pid,
2620 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2621 data->policy, data->rt_priority);
2622 seq_puts(m, "# -----------------\n");
2624 if (data->critical_start) {
2625 seq_puts(m, "# => started at: ");
2626 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2627 trace_print_seq(m, &iter->seq);
2628 seq_puts(m, "\n# => ended at: ");
2629 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2630 trace_print_seq(m, &iter->seq);
2631 seq_puts(m, "\n#\n");
2637 static void test_cpu_buff_start(struct trace_iterator *iter)
2639 struct trace_seq *s = &iter->seq;
2641 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2644 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2647 if (cpumask_test_cpu(iter->cpu, iter->started))
2650 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2653 cpumask_set_cpu(iter->cpu, iter->started);
2655 /* Don't print started cpu buffer for the first entry of the trace */
2657 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2661 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2663 struct trace_seq *s = &iter->seq;
2664 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2665 struct trace_entry *entry;
2666 struct trace_event *event;
2670 test_cpu_buff_start(iter);
2672 event = ftrace_find_event(entry->type);
2674 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2675 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2676 if (!trace_print_lat_context(iter))
2679 if (!trace_print_context(iter))
2685 return event->funcs->trace(iter, sym_flags, event);
2687 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2690 return TRACE_TYPE_HANDLED;
2692 return TRACE_TYPE_PARTIAL_LINE;
2695 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2697 struct trace_seq *s = &iter->seq;
2698 struct trace_entry *entry;
2699 struct trace_event *event;
2703 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2704 if (!trace_seq_printf(s, "%d %d %llu ",
2705 entry->pid, iter->cpu, iter->ts))
2709 event = ftrace_find_event(entry->type);
2711 return event->funcs->raw(iter, 0, event);
2713 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2716 return TRACE_TYPE_HANDLED;
2718 return TRACE_TYPE_PARTIAL_LINE;
2721 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2723 struct trace_seq *s = &iter->seq;
2724 unsigned char newline = '\n';
2725 struct trace_entry *entry;
2726 struct trace_event *event;
2730 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2731 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2732 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2733 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2736 event = ftrace_find_event(entry->type);
2738 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2739 if (ret != TRACE_TYPE_HANDLED)
2743 SEQ_PUT_FIELD_RET(s, newline);
2745 return TRACE_TYPE_HANDLED;
2748 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2750 struct trace_seq *s = &iter->seq;
2751 struct trace_entry *entry;
2752 struct trace_event *event;
2756 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2757 SEQ_PUT_FIELD_RET(s, entry->pid);
2758 SEQ_PUT_FIELD_RET(s, iter->cpu);
2759 SEQ_PUT_FIELD_RET(s, iter->ts);
2762 event = ftrace_find_event(entry->type);
2763 return event ? event->funcs->binary(iter, 0, event) :
2767 int trace_empty(struct trace_iterator *iter)
2769 struct ring_buffer_iter *buf_iter;
2772 /* If we are looking at one CPU buffer, only check that one */
2773 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2774 cpu = iter->cpu_file;
2775 buf_iter = trace_buffer_iter(iter, cpu);
2777 if (!ring_buffer_iter_empty(buf_iter))
2780 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2786 for_each_tracing_cpu(cpu) {
2787 buf_iter = trace_buffer_iter(iter, cpu);
2789 if (!ring_buffer_iter_empty(buf_iter))
2792 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2800 /* Called with trace_event_read_lock() held. */
2801 enum print_line_t print_trace_line(struct trace_iterator *iter)
2803 enum print_line_t ret;
2805 if (iter->lost_events &&
2806 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2807 iter->cpu, iter->lost_events))
2808 return TRACE_TYPE_PARTIAL_LINE;
2810 if (iter->trace && iter->trace->print_line) {
2811 ret = iter->trace->print_line(iter);
2812 if (ret != TRACE_TYPE_UNHANDLED)
2816 if (iter->ent->type == TRACE_BPUTS &&
2817 trace_flags & TRACE_ITER_PRINTK &&
2818 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2819 return trace_print_bputs_msg_only(iter);
2821 if (iter->ent->type == TRACE_BPRINT &&
2822 trace_flags & TRACE_ITER_PRINTK &&
2823 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2824 return trace_print_bprintk_msg_only(iter);
2826 if (iter->ent->type == TRACE_PRINT &&
2827 trace_flags & TRACE_ITER_PRINTK &&
2828 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2829 return trace_print_printk_msg_only(iter);
2831 if (trace_flags & TRACE_ITER_BIN)
2832 return print_bin_fmt(iter);
2834 if (trace_flags & TRACE_ITER_HEX)
2835 return print_hex_fmt(iter);
2837 if (trace_flags & TRACE_ITER_RAW)
2838 return print_raw_fmt(iter);
2840 return print_trace_fmt(iter);
2843 void trace_latency_header(struct seq_file *m)
2845 struct trace_iterator *iter = m->private;
2847 /* print nothing if the buffers are empty */
2848 if (trace_empty(iter))
2851 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2852 print_trace_header(m, iter);
2854 if (!(trace_flags & TRACE_ITER_VERBOSE))
2855 print_lat_help_header(m);
2858 void trace_default_header(struct seq_file *m)
2860 struct trace_iterator *iter = m->private;
2862 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2865 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2866 /* print nothing if the buffers are empty */
2867 if (trace_empty(iter))
2869 print_trace_header(m, iter);
2870 if (!(trace_flags & TRACE_ITER_VERBOSE))
2871 print_lat_help_header(m);
2873 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2874 if (trace_flags & TRACE_ITER_IRQ_INFO)
2875 print_func_help_header_irq(iter->trace_buffer, m);
2877 print_func_help_header(iter->trace_buffer, m);
2882 static void test_ftrace_alive(struct seq_file *m)
2884 if (!ftrace_is_dead())
2886 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2887 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2890 #ifdef CONFIG_TRACER_MAX_TRACE
2891 static void show_snapshot_main_help(struct seq_file *m)
2893 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2894 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2895 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2896 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2897 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2898 seq_printf(m, "# is not a '0' or '1')\n");
2901 static void show_snapshot_percpu_help(struct seq_file *m)
2903 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2904 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2905 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2906 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2908 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2909 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2911 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2912 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2913 seq_printf(m, "# is not a '0' or '1')\n");
2916 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2918 if (iter->tr->allocated_snapshot)
2919 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2921 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2923 seq_printf(m, "# Snapshot commands:\n");
2924 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2925 show_snapshot_main_help(m);
2927 show_snapshot_percpu_help(m);
2930 /* Should never be called */
2931 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2934 static int s_show(struct seq_file *m, void *v)
2936 struct trace_iterator *iter = v;
2939 if (iter->ent == NULL) {
2941 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2943 test_ftrace_alive(m);
2945 if (iter->snapshot && trace_empty(iter))
2946 print_snapshot_help(m, iter);
2947 else if (iter->trace && iter->trace->print_header)
2948 iter->trace->print_header(m);
2950 trace_default_header(m);
2952 } else if (iter->leftover) {
2954 * If we filled the seq_file buffer earlier, we
2955 * want to just show it now.
2957 ret = trace_print_seq(m, &iter->seq);
2959 /* ret should this time be zero, but you never know */
2960 iter->leftover = ret;
2963 print_trace_line(iter);
2964 ret = trace_print_seq(m, &iter->seq);
2966 * If we overflow the seq_file buffer, then it will
2967 * ask us for this data again at start up.
2969 * ret is 0 if seq_file write succeeded.
2972 iter->leftover = ret;
2979 * Should be used after trace_array_get(), trace_types_lock
2980 * ensures that i_cdev was already initialized.
2982 static inline int tracing_get_cpu(struct inode *inode)
2984 if (inode->i_cdev) /* See trace_create_cpu_file() */
2985 return (long)inode->i_cdev - 1;
2986 return RING_BUFFER_ALL_CPUS;
2989 static const struct seq_operations tracer_seq_ops = {
2996 static struct trace_iterator *
2997 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2999 struct trace_array *tr = inode->i_private;
3000 struct trace_iterator *iter;
3003 if (tracing_disabled)
3004 return ERR_PTR(-ENODEV);
3006 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3008 return ERR_PTR(-ENOMEM);
3010 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
3012 if (!iter->buffer_iter)
3016 * We make a copy of the current tracer to avoid concurrent
3017 * changes on it while we are reading.
3019 mutex_lock(&trace_types_lock);
3020 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3024 *iter->trace = *tr->current_trace;
3026 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3031 #ifdef CONFIG_TRACER_MAX_TRACE
3032 /* Currently only the top directory has a snapshot */
3033 if (tr->current_trace->print_max || snapshot)
3034 iter->trace_buffer = &tr->max_buffer;
3037 iter->trace_buffer = &tr->trace_buffer;
3038 iter->snapshot = snapshot;
3040 iter->cpu_file = tracing_get_cpu(inode);
3041 mutex_init(&iter->mutex);
3043 /* Notify the tracer early; before we stop tracing. */
3044 if (iter->trace && iter->trace->open)
3045 iter->trace->open(iter);
3047 /* Annotate start of buffers if we had overruns */
3048 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3049 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3051 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3052 if (trace_clocks[tr->clock_id].in_ns)
3053 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3055 /* stop the trace while dumping if we are not opening "snapshot" */
3056 if (!iter->snapshot)
3057 tracing_stop_tr(tr);
3059 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3060 for_each_tracing_cpu(cpu) {
3061 iter->buffer_iter[cpu] =
3062 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3064 ring_buffer_read_prepare_sync();
3065 for_each_tracing_cpu(cpu) {
3066 ring_buffer_read_start(iter->buffer_iter[cpu]);
3067 tracing_iter_reset(iter, cpu);
3070 cpu = iter->cpu_file;
3071 iter->buffer_iter[cpu] =
3072 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3073 ring_buffer_read_prepare_sync();
3074 ring_buffer_read_start(iter->buffer_iter[cpu]);
3075 tracing_iter_reset(iter, cpu);
3078 mutex_unlock(&trace_types_lock);
3083 mutex_unlock(&trace_types_lock);
3085 kfree(iter->buffer_iter);
3087 seq_release_private(inode, file);
3088 return ERR_PTR(-ENOMEM);
3091 int tracing_open_generic(struct inode *inode, struct file *filp)
3093 if (tracing_disabled)
3096 filp->private_data = inode->i_private;
3100 bool tracing_is_disabled(void)
3102 return (tracing_disabled) ? true: false;
3106 * Open and update trace_array ref count.
3107 * Must have the current trace_array passed to it.
3109 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3111 struct trace_array *tr = inode->i_private;
3113 if (tracing_disabled)
3116 if (trace_array_get(tr) < 0)
3119 filp->private_data = inode->i_private;
3124 static int tracing_release(struct inode *inode, struct file *file)
3126 struct trace_array *tr = inode->i_private;
3127 struct seq_file *m = file->private_data;
3128 struct trace_iterator *iter;
3131 if (!(file->f_mode & FMODE_READ)) {
3132 trace_array_put(tr);
3136 /* Writes do not use seq_file */
3138 mutex_lock(&trace_types_lock);
3140 for_each_tracing_cpu(cpu) {
3141 if (iter->buffer_iter[cpu])
3142 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3145 if (iter->trace && iter->trace->close)
3146 iter->trace->close(iter);
3148 if (!iter->snapshot)
3149 /* reenable tracing if it was previously enabled */
3150 tracing_start_tr(tr);
3152 __trace_array_put(tr);
3154 mutex_unlock(&trace_types_lock);
3156 mutex_destroy(&iter->mutex);
3157 free_cpumask_var(iter->started);
3159 kfree(iter->buffer_iter);
3160 seq_release_private(inode, file);
3165 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3167 struct trace_array *tr = inode->i_private;
3169 trace_array_put(tr);
3173 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3175 struct trace_array *tr = inode->i_private;
3177 trace_array_put(tr);
3179 return single_release(inode, file);
3182 static int tracing_open(struct inode *inode, struct file *file)
3184 struct trace_array *tr = inode->i_private;
3185 struct trace_iterator *iter;
3188 if (trace_array_get(tr) < 0)
3191 /* If this file was open for write, then erase contents */
3192 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3193 int cpu = tracing_get_cpu(inode);
3195 if (cpu == RING_BUFFER_ALL_CPUS)
3196 tracing_reset_online_cpus(&tr->trace_buffer);
3198 tracing_reset(&tr->trace_buffer, cpu);
3201 if (file->f_mode & FMODE_READ) {
3202 iter = __tracing_open(inode, file, false);
3204 ret = PTR_ERR(iter);
3205 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3206 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3210 trace_array_put(tr);
3216 * Some tracers are not suitable for instance buffers.
3217 * A tracer is always available for the global array (toplevel)
3218 * or if it explicitly states that it is.
3221 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3223 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3226 /* Find the next tracer that this trace array may use */
3227 static struct tracer *
3228 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3230 while (t && !trace_ok_for_array(t, tr))
3237 t_next(struct seq_file *m, void *v, loff_t *pos)
3239 struct trace_array *tr = m->private;
3240 struct tracer *t = v;
3245 t = get_tracer_for_array(tr, t->next);
3250 static void *t_start(struct seq_file *m, loff_t *pos)
3252 struct trace_array *tr = m->private;
3256 mutex_lock(&trace_types_lock);
3258 t = get_tracer_for_array(tr, trace_types);
3259 for (; t && l < *pos; t = t_next(m, t, &l))
3265 static void t_stop(struct seq_file *m, void *p)
3267 mutex_unlock(&trace_types_lock);
3270 static int t_show(struct seq_file *m, void *v)
3272 struct tracer *t = v;
3277 seq_printf(m, "%s", t->name);
3286 static const struct seq_operations show_traces_seq_ops = {
3293 static int show_traces_open(struct inode *inode, struct file *file)
3295 struct trace_array *tr = inode->i_private;
3299 if (tracing_disabled)
3302 ret = seq_open(file, &show_traces_seq_ops);
3306 m = file->private_data;
3313 tracing_write_stub(struct file *filp, const char __user *ubuf,
3314 size_t count, loff_t *ppos)
3319 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3323 if (file->f_mode & FMODE_READ)
3324 ret = seq_lseek(file, offset, whence);
3326 file->f_pos = ret = 0;
3331 static const struct file_operations tracing_fops = {
3332 .open = tracing_open,
3334 .write = tracing_write_stub,
3335 .llseek = tracing_lseek,
3336 .release = tracing_release,
3339 static const struct file_operations show_traces_fops = {
3340 .open = show_traces_open,
3342 .release = seq_release,
3343 .llseek = seq_lseek,
3347 * The tracer itself will not take this lock, but still we want
3348 * to provide a consistent cpumask to user-space:
3350 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3353 * Temporary storage for the character representation of the
3354 * CPU bitmask (and one more byte for the newline):
3356 static char mask_str[NR_CPUS + 1];
3359 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3360 size_t count, loff_t *ppos)
3362 struct trace_array *tr = file_inode(filp)->i_private;
3365 mutex_lock(&tracing_cpumask_update_lock);
3367 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3368 if (count - len < 2) {
3372 len += sprintf(mask_str + len, "\n");
3373 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3376 mutex_unlock(&tracing_cpumask_update_lock);
3382 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3383 size_t count, loff_t *ppos)
3385 struct trace_array *tr = file_inode(filp)->i_private;
3386 cpumask_var_t tracing_cpumask_new;
3389 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3392 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3396 mutex_lock(&tracing_cpumask_update_lock);
3398 local_irq_disable();
3399 arch_spin_lock(&tr->max_lock);
3400 for_each_tracing_cpu(cpu) {
3402 * Increase/decrease the disabled counter if we are
3403 * about to flip a bit in the cpumask:
3405 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3406 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3407 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3408 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3410 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3411 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3412 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3413 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3416 arch_spin_unlock(&tr->max_lock);
3419 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3421 mutex_unlock(&tracing_cpumask_update_lock);
3422 free_cpumask_var(tracing_cpumask_new);
3427 free_cpumask_var(tracing_cpumask_new);
3432 static const struct file_operations tracing_cpumask_fops = {
3433 .open = tracing_open_generic_tr,
3434 .read = tracing_cpumask_read,
3435 .write = tracing_cpumask_write,
3436 .release = tracing_release_generic_tr,
3437 .llseek = generic_file_llseek,
3440 static int tracing_trace_options_show(struct seq_file *m, void *v)
3442 struct tracer_opt *trace_opts;
3443 struct trace_array *tr = m->private;
3447 mutex_lock(&trace_types_lock);
3448 tracer_flags = tr->current_trace->flags->val;
3449 trace_opts = tr->current_trace->flags->opts;
3451 for (i = 0; trace_options[i]; i++) {
3452 if (trace_flags & (1 << i))
3453 seq_printf(m, "%s\n", trace_options[i]);
3455 seq_printf(m, "no%s\n", trace_options[i]);
3458 for (i = 0; trace_opts[i].name; i++) {
3459 if (tracer_flags & trace_opts[i].bit)
3460 seq_printf(m, "%s\n", trace_opts[i].name);
3462 seq_printf(m, "no%s\n", trace_opts[i].name);
3464 mutex_unlock(&trace_types_lock);
3469 static int __set_tracer_option(struct trace_array *tr,
3470 struct tracer_flags *tracer_flags,
3471 struct tracer_opt *opts, int neg)
3473 struct tracer *trace = tr->current_trace;
3476 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3481 tracer_flags->val &= ~opts->bit;
3483 tracer_flags->val |= opts->bit;
3487 /* Try to assign a tracer specific option */
3488 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3490 struct tracer *trace = tr->current_trace;
3491 struct tracer_flags *tracer_flags = trace->flags;
3492 struct tracer_opt *opts = NULL;
3495 for (i = 0; tracer_flags->opts[i].name; i++) {
3496 opts = &tracer_flags->opts[i];
3498 if (strcmp(cmp, opts->name) == 0)
3499 return __set_tracer_option(tr, trace->flags, opts, neg);
3505 /* Some tracers require overwrite to stay enabled */
3506 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3508 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3514 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3516 /* do nothing if flag is already set */
3517 if (!!(trace_flags & mask) == !!enabled)
3520 /* Give the tracer a chance to approve the change */
3521 if (tr->current_trace->flag_changed)
3522 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3526 trace_flags |= mask;
3528 trace_flags &= ~mask;
3530 if (mask == TRACE_ITER_RECORD_CMD)
3531 trace_event_enable_cmd_record(enabled);
3533 if (mask == TRACE_ITER_OVERWRITE) {
3534 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3535 #ifdef CONFIG_TRACER_MAX_TRACE
3536 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3540 if (mask == TRACE_ITER_PRINTK)
3541 trace_printk_start_stop_comm(enabled);
3546 static int trace_set_options(struct trace_array *tr, char *option)
3553 cmp = strstrip(option);
3555 if (strncmp(cmp, "no", 2) == 0) {
3560 mutex_lock(&trace_types_lock);
3562 for (i = 0; trace_options[i]; i++) {
3563 if (strcmp(cmp, trace_options[i]) == 0) {
3564 ret = set_tracer_flag(tr, 1 << i, !neg);
3569 /* If no option could be set, test the specific tracer options */
3570 if (!trace_options[i])
3571 ret = set_tracer_option(tr, cmp, neg);
3573 mutex_unlock(&trace_types_lock);
3579 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3580 size_t cnt, loff_t *ppos)
3582 struct seq_file *m = filp->private_data;
3583 struct trace_array *tr = m->private;
3587 if (cnt >= sizeof(buf))
3590 if (copy_from_user(&buf, ubuf, cnt))
3595 ret = trace_set_options(tr, buf);
3604 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3606 struct trace_array *tr = inode->i_private;
3609 if (tracing_disabled)
3612 if (trace_array_get(tr) < 0)
3615 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3617 trace_array_put(tr);
3622 static const struct file_operations tracing_iter_fops = {
3623 .open = tracing_trace_options_open,
3625 .llseek = seq_lseek,
3626 .release = tracing_single_release_tr,
3627 .write = tracing_trace_options_write,
3630 static const char readme_msg[] =
3631 "tracing mini-HOWTO:\n\n"
3632 "# echo 0 > tracing_on : quick way to disable tracing\n"
3633 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3634 " Important files:\n"
3635 " trace\t\t\t- The static contents of the buffer\n"
3636 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3637 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3638 " current_tracer\t- function and latency tracers\n"
3639 " available_tracers\t- list of configured tracers for current_tracer\n"
3640 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3641 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3642 " trace_clock\t\t-change the clock used to order events\n"
3643 " local: Per cpu clock but may not be synced across CPUs\n"
3644 " global: Synced across CPUs but slows tracing down.\n"
3645 " counter: Not a clock, but just an increment\n"
3646 " uptime: Jiffy counter from time of boot\n"
3647 " perf: Same clock that perf events use\n"
3648 #ifdef CONFIG_X86_64
3649 " x86-tsc: TSC cycle counter\n"
3651 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3652 " tracing_cpumask\t- Limit which CPUs to trace\n"
3653 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3654 "\t\t\t Remove sub-buffer with rmdir\n"
3655 " trace_options\t\t- Set format or modify how tracing happens\n"
3656 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3657 "\t\t\t option name\n"
3658 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3659 #ifdef CONFIG_DYNAMIC_FTRACE
3660 "\n available_filter_functions - list of functions that can be filtered on\n"
3661 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3662 "\t\t\t functions\n"
3663 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3664 "\t modules: Can select a group via module\n"
3665 "\t Format: :mod:<module-name>\n"
3666 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3667 "\t triggers: a command to perform when function is hit\n"
3668 "\t Format: <function>:<trigger>[:count]\n"
3669 "\t trigger: traceon, traceoff\n"
3670 "\t\t enable_event:<system>:<event>\n"
3671 "\t\t disable_event:<system>:<event>\n"
3672 #ifdef CONFIG_STACKTRACE
3675 #ifdef CONFIG_TRACER_SNAPSHOT
3680 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3681 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3682 "\t The first one will disable tracing every time do_fault is hit\n"
3683 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3684 "\t The first time do trap is hit and it disables tracing, the\n"
3685 "\t counter will decrement to 2. If tracing is already disabled,\n"
3686 "\t the counter will not decrement. It only decrements when the\n"
3687 "\t trigger did work\n"
3688 "\t To remove trigger without count:\n"
3689 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3690 "\t To remove trigger with a count:\n"
3691 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3692 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3693 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3694 "\t modules: Can select a group via module command :mod:\n"
3695 "\t Does not accept triggers\n"
3696 #endif /* CONFIG_DYNAMIC_FTRACE */
3697 #ifdef CONFIG_FUNCTION_TRACER
3698 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3701 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3702 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3703 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3705 #ifdef CONFIG_TRACER_SNAPSHOT
3706 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3707 "\t\t\t snapshot buffer. Read the contents for more\n"
3708 "\t\t\t information\n"
3710 #ifdef CONFIG_STACK_TRACER
3711 " stack_trace\t\t- Shows the max stack trace when active\n"
3712 " stack_max_size\t- Shows current max stack size that was traced\n"
3713 "\t\t\t Write into this file to reset the max size (trigger a\n"
3714 "\t\t\t new trace)\n"
3715 #ifdef CONFIG_DYNAMIC_FTRACE
3716 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3719 #endif /* CONFIG_STACK_TRACER */
3720 " events/\t\t- Directory containing all trace event subsystems:\n"
3721 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3722 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3723 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3725 " filter\t\t- If set, only events passing filter are traced\n"
3726 " events/<system>/<event>/\t- Directory containing control files for\n"
3728 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3729 " filter\t\t- If set, only events passing filter are traced\n"
3730 " trigger\t\t- If set, a command to perform when event is hit\n"
3731 "\t Format: <trigger>[:count][if <filter>]\n"
3732 "\t trigger: traceon, traceoff\n"
3733 "\t enable_event:<system>:<event>\n"
3734 "\t disable_event:<system>:<event>\n"
3735 #ifdef CONFIG_STACKTRACE
3738 #ifdef CONFIG_TRACER_SNAPSHOT
3741 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3742 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3743 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3744 "\t events/block/block_unplug/trigger\n"
3745 "\t The first disables tracing every time block_unplug is hit.\n"
3746 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3747 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3748 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3749 "\t Like function triggers, the counter is only decremented if it\n"
3750 "\t enabled or disabled tracing.\n"
3751 "\t To remove a trigger without a count:\n"
3752 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3753 "\t To remove a trigger with a count:\n"
3754 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3755 "\t Filters can be ignored when removing a trigger.\n"
3759 tracing_readme_read(struct file *filp, char __user *ubuf,
3760 size_t cnt, loff_t *ppos)
3762 return simple_read_from_buffer(ubuf, cnt, ppos,
3763 readme_msg, strlen(readme_msg));
3766 static const struct file_operations tracing_readme_fops = {
3767 .open = tracing_open_generic,
3768 .read = tracing_readme_read,
3769 .llseek = generic_file_llseek,
3772 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3774 unsigned int *ptr = v;
3776 if (*pos || m->count)
3781 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3783 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3792 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3798 arch_spin_lock(&trace_cmdline_lock);
3800 v = &savedcmd->map_cmdline_to_pid[0];
3802 v = saved_cmdlines_next(m, v, &l);
3810 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3812 arch_spin_unlock(&trace_cmdline_lock);
3816 static int saved_cmdlines_show(struct seq_file *m, void *v)
3818 char buf[TASK_COMM_LEN];
3819 unsigned int *pid = v;
3821 __trace_find_cmdline(*pid, buf);
3822 seq_printf(m, "%d %s\n", *pid, buf);
3826 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3827 .start = saved_cmdlines_start,
3828 .next = saved_cmdlines_next,
3829 .stop = saved_cmdlines_stop,
3830 .show = saved_cmdlines_show,
3833 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3835 if (tracing_disabled)
3838 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3841 static const struct file_operations tracing_saved_cmdlines_fops = {
3842 .open = tracing_saved_cmdlines_open,
3844 .llseek = seq_lseek,
3845 .release = seq_release,
3849 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3850 size_t cnt, loff_t *ppos)
3855 arch_spin_lock(&trace_cmdline_lock);
3856 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3857 arch_spin_unlock(&trace_cmdline_lock);
3859 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3862 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3864 kfree(s->saved_cmdlines);
3865 kfree(s->map_cmdline_to_pid);
3869 static int tracing_resize_saved_cmdlines(unsigned int val)
3871 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3873 s = kmalloc(sizeof(*s), GFP_KERNEL);
3877 if (allocate_cmdlines_buffer(val, s) < 0) {
3882 arch_spin_lock(&trace_cmdline_lock);
3883 savedcmd_temp = savedcmd;
3885 arch_spin_unlock(&trace_cmdline_lock);
3886 free_saved_cmdlines_buffer(savedcmd_temp);
3892 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3893 size_t cnt, loff_t *ppos)
3898 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3902 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3903 if (!val || val > PID_MAX_DEFAULT)
3906 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3915 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3916 .open = tracing_open_generic,
3917 .read = tracing_saved_cmdlines_size_read,
3918 .write = tracing_saved_cmdlines_size_write,
3922 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3923 size_t cnt, loff_t *ppos)
3925 struct trace_array *tr = filp->private_data;
3926 char buf[MAX_TRACER_SIZE+2];
3929 mutex_lock(&trace_types_lock);
3930 r = sprintf(buf, "%s\n", tr->current_trace->name);
3931 mutex_unlock(&trace_types_lock);
3933 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3936 int tracer_init(struct tracer *t, struct trace_array *tr)
3938 tracing_reset_online_cpus(&tr->trace_buffer);
3942 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3946 for_each_tracing_cpu(cpu)
3947 per_cpu_ptr(buf->data, cpu)->entries = val;
3950 #ifdef CONFIG_TRACER_MAX_TRACE
3951 /* resize @tr's buffer to the size of @size_tr's entries */
3952 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3953 struct trace_buffer *size_buf, int cpu_id)
3957 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3958 for_each_tracing_cpu(cpu) {
3959 ret = ring_buffer_resize(trace_buf->buffer,
3960 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3963 per_cpu_ptr(trace_buf->data, cpu)->entries =
3964 per_cpu_ptr(size_buf->data, cpu)->entries;
3967 ret = ring_buffer_resize(trace_buf->buffer,
3968 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3970 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3971 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3976 #endif /* CONFIG_TRACER_MAX_TRACE */
3978 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3979 unsigned long size, int cpu)
3984 * If kernel or user changes the size of the ring buffer
3985 * we use the size that was given, and we can forget about
3986 * expanding it later.
3988 ring_buffer_expanded = true;
3990 /* May be called before buffers are initialized */
3991 if (!tr->trace_buffer.buffer)
3994 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3998 #ifdef CONFIG_TRACER_MAX_TRACE
3999 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4000 !tr->current_trace->use_max_tr)
4003 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4005 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4006 &tr->trace_buffer, cpu);
4009 * AARGH! We are left with different
4010 * size max buffer!!!!
4011 * The max buffer is our "snapshot" buffer.
4012 * When a tracer needs a snapshot (one of the
4013 * latency tracers), it swaps the max buffer
4014 * with the saved snap shot. We succeeded to
4015 * update the size of the main buffer, but failed to
4016 * update the size of the max buffer. But when we tried
4017 * to reset the main buffer to the original size, we
4018 * failed there too. This is very unlikely to
4019 * happen, but if it does, warn and kill all
4023 tracing_disabled = 1;
4028 if (cpu == RING_BUFFER_ALL_CPUS)
4029 set_buffer_entries(&tr->max_buffer, size);
4031 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4034 #endif /* CONFIG_TRACER_MAX_TRACE */
4036 if (cpu == RING_BUFFER_ALL_CPUS)
4037 set_buffer_entries(&tr->trace_buffer, size);
4039 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4044 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4045 unsigned long size, int cpu_id)
4049 mutex_lock(&trace_types_lock);
4051 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4052 /* make sure, this cpu is enabled in the mask */
4053 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4059 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4064 mutex_unlock(&trace_types_lock);
4071 * tracing_update_buffers - used by tracing facility to expand ring buffers
4073 * To save on memory when the tracing is never used on a system with it
4074 * configured in. The ring buffers are set to a minimum size. But once
4075 * a user starts to use the tracing facility, then they need to grow
4076 * to their default size.
4078 * This function is to be called when a tracer is about to be used.
4080 int tracing_update_buffers(void)
4084 mutex_lock(&trace_types_lock);
4085 if (!ring_buffer_expanded)
4086 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4087 RING_BUFFER_ALL_CPUS);
4088 mutex_unlock(&trace_types_lock);
4093 struct trace_option_dentry;
4095 static struct trace_option_dentry *
4096 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4099 destroy_trace_option_files(struct trace_option_dentry *topts);
4102 * Used to clear out the tracer before deletion of an instance.
4103 * Must have trace_types_lock held.
4105 static void tracing_set_nop(struct trace_array *tr)
4107 if (tr->current_trace == &nop_trace)
4110 tr->current_trace->enabled--;
4112 if (tr->current_trace->reset)
4113 tr->current_trace->reset(tr);
4115 tr->current_trace = &nop_trace;
4118 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4120 static struct trace_option_dentry *topts;
4122 #ifdef CONFIG_TRACER_MAX_TRACE
4127 mutex_lock(&trace_types_lock);
4129 if (!ring_buffer_expanded) {
4130 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4131 RING_BUFFER_ALL_CPUS);
4137 for (t = trace_types; t; t = t->next) {
4138 if (strcmp(t->name, buf) == 0)
4145 if (t == tr->current_trace)
4148 /* Some tracers are only allowed for the top level buffer */
4149 if (!trace_ok_for_array(t, tr)) {
4154 trace_branch_disable();
4156 tr->current_trace->enabled--;
4158 if (tr->current_trace->reset)
4159 tr->current_trace->reset(tr);
4161 /* Current trace needs to be nop_trace before synchronize_sched */
4162 tr->current_trace = &nop_trace;
4164 #ifdef CONFIG_TRACER_MAX_TRACE
4165 had_max_tr = tr->allocated_snapshot;
4167 if (had_max_tr && !t->use_max_tr) {
4169 * We need to make sure that the update_max_tr sees that
4170 * current_trace changed to nop_trace to keep it from
4171 * swapping the buffers after we resize it.
4172 * The update_max_tr is called from interrupts disabled
4173 * so a synchronized_sched() is sufficient.
4175 synchronize_sched();
4179 /* Currently, only the top instance has options */
4180 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4181 destroy_trace_option_files(topts);
4182 topts = create_trace_option_files(tr, t);
4185 #ifdef CONFIG_TRACER_MAX_TRACE
4186 if (t->use_max_tr && !had_max_tr) {
4187 ret = alloc_snapshot(tr);
4194 ret = tracer_init(t, tr);
4199 tr->current_trace = t;
4200 tr->current_trace->enabled++;
4201 trace_branch_enable(tr);
4203 mutex_unlock(&trace_types_lock);
4209 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4210 size_t cnt, loff_t *ppos)
4212 struct trace_array *tr = filp->private_data;
4213 char buf[MAX_TRACER_SIZE+1];
4220 if (cnt > MAX_TRACER_SIZE)
4221 cnt = MAX_TRACER_SIZE;
4223 if (copy_from_user(&buf, ubuf, cnt))
4228 /* strip ending whitespace. */
4229 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4232 err = tracing_set_tracer(tr, buf);
4242 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4243 size_t cnt, loff_t *ppos)
4245 unsigned long *ptr = filp->private_data;
4249 r = snprintf(buf, sizeof(buf), "%ld\n",
4250 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4251 if (r > sizeof(buf))
4253 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4257 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4258 size_t cnt, loff_t *ppos)
4260 unsigned long *ptr = filp->private_data;
4264 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4273 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4275 struct trace_array *tr = inode->i_private;
4276 struct trace_iterator *iter;
4279 if (tracing_disabled)
4282 if (trace_array_get(tr) < 0)
4285 mutex_lock(&trace_types_lock);
4287 /* create a buffer to store the information to pass to userspace */
4288 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4291 __trace_array_put(tr);
4296 * We make a copy of the current tracer to avoid concurrent
4297 * changes on it while we are reading.
4299 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4304 *iter->trace = *tr->current_trace;
4306 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4311 /* trace pipe does not show start of buffer */
4312 cpumask_setall(iter->started);
4314 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4315 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4317 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4318 if (trace_clocks[tr->clock_id].in_ns)
4319 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4322 iter->trace_buffer = &tr->trace_buffer;
4323 iter->cpu_file = tracing_get_cpu(inode);
4324 mutex_init(&iter->mutex);
4325 filp->private_data = iter;
4327 if (iter->trace->pipe_open)
4328 iter->trace->pipe_open(iter);
4330 nonseekable_open(inode, filp);
4332 mutex_unlock(&trace_types_lock);
4338 __trace_array_put(tr);
4339 mutex_unlock(&trace_types_lock);
4343 static int tracing_release_pipe(struct inode *inode, struct file *file)
4345 struct trace_iterator *iter = file->private_data;
4346 struct trace_array *tr = inode->i_private;
4348 mutex_lock(&trace_types_lock);
4350 if (iter->trace->pipe_close)
4351 iter->trace->pipe_close(iter);
4353 mutex_unlock(&trace_types_lock);
4355 free_cpumask_var(iter->started);
4356 mutex_destroy(&iter->mutex);
4360 trace_array_put(tr);
4366 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4368 /* Iterators are static, they should be filled or empty */
4369 if (trace_buffer_iter(iter, iter->cpu_file))
4370 return POLLIN | POLLRDNORM;
4372 if (trace_flags & TRACE_ITER_BLOCK)
4374 * Always select as readable when in blocking mode
4376 return POLLIN | POLLRDNORM;
4378 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4383 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4385 struct trace_iterator *iter = filp->private_data;
4387 return trace_poll(iter, filp, poll_table);
4390 /* Must be called with trace_types_lock mutex held. */
4391 static int tracing_wait_pipe(struct file *filp)
4393 struct trace_iterator *iter = filp->private_data;
4396 while (trace_empty(iter)) {
4398 if ((filp->f_flags & O_NONBLOCK)) {
4403 * We block until we read something and tracing is disabled.
4404 * We still block if tracing is disabled, but we have never
4405 * read anything. This allows a user to cat this file, and
4406 * then enable tracing. But after we have read something,
4407 * we give an EOF when tracing is again disabled.
4409 * iter->pos will be 0 if we haven't read anything.
4411 if (!tracing_is_on() && iter->pos)
4414 mutex_unlock(&iter->mutex);
4416 ret = wait_on_pipe(iter, false);
4418 mutex_lock(&iter->mutex);
4431 tracing_read_pipe(struct file *filp, char __user *ubuf,
4432 size_t cnt, loff_t *ppos)
4434 struct trace_iterator *iter = filp->private_data;
4435 struct trace_array *tr = iter->tr;
4438 /* copy the tracer to avoid using a global lock all around */
4439 mutex_lock(&trace_types_lock);
4440 if (unlikely(iter->trace->name != tr->current_trace->name))
4441 *iter->trace = *tr->current_trace;
4442 mutex_unlock(&trace_types_lock);
4445 * Avoid more than one consumer on a single file descriptor
4446 * This is just a matter of traces coherency, the ring buffer itself
4449 mutex_lock(&iter->mutex);
4451 /* return any leftover data */
4452 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4456 trace_seq_init(&iter->seq);
4458 if (iter->trace->read) {
4459 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4465 sret = tracing_wait_pipe(filp);
4469 /* stop when tracing is finished */
4470 if (trace_empty(iter)) {
4475 if (cnt >= PAGE_SIZE)
4476 cnt = PAGE_SIZE - 1;
4478 /* reset all but tr, trace, and overruns */
4479 memset(&iter->seq, 0,
4480 sizeof(struct trace_iterator) -
4481 offsetof(struct trace_iterator, seq));
4482 cpumask_clear(iter->started);
4485 trace_event_read_lock();
4486 trace_access_lock(iter->cpu_file);
4487 while (trace_find_next_entry_inc(iter) != NULL) {
4488 enum print_line_t ret;
4489 int len = iter->seq.len;
4491 ret = print_trace_line(iter);
4492 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4493 /* don't print partial lines */
4494 iter->seq.len = len;
4497 if (ret != TRACE_TYPE_NO_CONSUME)
4498 trace_consume(iter);
4500 if (iter->seq.len >= cnt)
4504 * Setting the full flag means we reached the trace_seq buffer
4505 * size and we should leave by partial output condition above.
4506 * One of the trace_seq_* functions is not used properly.
4508 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4511 trace_access_unlock(iter->cpu_file);
4512 trace_event_read_unlock();
4514 /* Now copy what we have to the user */
4515 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4516 if (iter->seq.readpos >= iter->seq.len)
4517 trace_seq_init(&iter->seq);
4520 * If there was nothing to send to user, in spite of consuming trace
4521 * entries, go back to wait for more entries.
4527 mutex_unlock(&iter->mutex);
4532 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4535 __free_page(spd->pages[idx]);
4538 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4540 .confirm = generic_pipe_buf_confirm,
4541 .release = generic_pipe_buf_release,
4542 .steal = generic_pipe_buf_steal,
4543 .get = generic_pipe_buf_get,
4547 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4552 /* Seq buffer is page-sized, exactly what we need. */
4554 count = iter->seq.len;
4555 ret = print_trace_line(iter);
4556 count = iter->seq.len - count;
4559 iter->seq.len -= count;
4562 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4563 iter->seq.len -= count;
4567 if (ret != TRACE_TYPE_NO_CONSUME)
4568 trace_consume(iter);
4570 if (!trace_find_next_entry_inc(iter)) {
4580 static ssize_t tracing_splice_read_pipe(struct file *filp,
4582 struct pipe_inode_info *pipe,
4586 struct page *pages_def[PIPE_DEF_BUFFERS];
4587 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4588 struct trace_iterator *iter = filp->private_data;
4589 struct splice_pipe_desc spd = {
4591 .partial = partial_def,
4592 .nr_pages = 0, /* This gets updated below. */
4593 .nr_pages_max = PIPE_DEF_BUFFERS,
4595 .ops = &tracing_pipe_buf_ops,
4596 .spd_release = tracing_spd_release_pipe,
4598 struct trace_array *tr = iter->tr;
4603 if (splice_grow_spd(pipe, &spd))
4606 /* copy the tracer to avoid using a global lock all around */
4607 mutex_lock(&trace_types_lock);
4608 if (unlikely(iter->trace->name != tr->current_trace->name))
4609 *iter->trace = *tr->current_trace;
4610 mutex_unlock(&trace_types_lock);
4612 mutex_lock(&iter->mutex);
4614 if (iter->trace->splice_read) {
4615 ret = iter->trace->splice_read(iter, filp,
4616 ppos, pipe, len, flags);
4621 ret = tracing_wait_pipe(filp);
4625 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4630 trace_event_read_lock();
4631 trace_access_lock(iter->cpu_file);
4633 /* Fill as many pages as possible. */
4634 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4635 spd.pages[i] = alloc_page(GFP_KERNEL);
4639 rem = tracing_fill_pipe_page(rem, iter);
4641 /* Copy the data into the page, so we can start over. */
4642 ret = trace_seq_to_buffer(&iter->seq,
4643 page_address(spd.pages[i]),
4646 __free_page(spd.pages[i]);
4649 spd.partial[i].offset = 0;
4650 spd.partial[i].len = iter->seq.len;
4652 trace_seq_init(&iter->seq);
4655 trace_access_unlock(iter->cpu_file);
4656 trace_event_read_unlock();
4657 mutex_unlock(&iter->mutex);
4662 ret = splice_to_pipe(pipe, &spd);
4666 splice_shrink_spd(&spd);
4670 mutex_unlock(&iter->mutex);
4675 tracing_entries_read(struct file *filp, char __user *ubuf,
4676 size_t cnt, loff_t *ppos)
4678 struct inode *inode = file_inode(filp);
4679 struct trace_array *tr = inode->i_private;
4680 int cpu = tracing_get_cpu(inode);
4685 mutex_lock(&trace_types_lock);
4687 if (cpu == RING_BUFFER_ALL_CPUS) {
4688 int cpu, buf_size_same;
4693 /* check if all cpu sizes are same */
4694 for_each_tracing_cpu(cpu) {
4695 /* fill in the size from first enabled cpu */
4697 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4698 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4704 if (buf_size_same) {
4705 if (!ring_buffer_expanded)
4706 r = sprintf(buf, "%lu (expanded: %lu)\n",
4708 trace_buf_size >> 10);
4710 r = sprintf(buf, "%lu\n", size >> 10);
4712 r = sprintf(buf, "X\n");
4714 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4716 mutex_unlock(&trace_types_lock);
4718 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4723 tracing_entries_write(struct file *filp, const char __user *ubuf,
4724 size_t cnt, loff_t *ppos)
4726 struct inode *inode = file_inode(filp);
4727 struct trace_array *tr = inode->i_private;
4731 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4735 /* must have at least 1 entry */
4739 /* value is in KB */
4741 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4751 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4752 size_t cnt, loff_t *ppos)
4754 struct trace_array *tr = filp->private_data;
4757 unsigned long size = 0, expanded_size = 0;
4759 mutex_lock(&trace_types_lock);
4760 for_each_tracing_cpu(cpu) {
4761 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4762 if (!ring_buffer_expanded)
4763 expanded_size += trace_buf_size >> 10;
4765 if (ring_buffer_expanded)
4766 r = sprintf(buf, "%lu\n", size);
4768 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4769 mutex_unlock(&trace_types_lock);
4771 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4775 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4776 size_t cnt, loff_t *ppos)
4779 * There is no need to read what the user has written, this function
4780 * is just to make sure that there is no error when "echo" is used
4789 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4791 struct trace_array *tr = inode->i_private;
4793 /* disable tracing ? */
4794 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4795 tracer_tracing_off(tr);
4796 /* resize the ring buffer to 0 */
4797 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4799 trace_array_put(tr);
4805 tracing_mark_write(struct file *filp, const char __user *ubuf,
4806 size_t cnt, loff_t *fpos)
4808 unsigned long addr = (unsigned long)ubuf;
4809 struct trace_array *tr = filp->private_data;
4810 struct ring_buffer_event *event;
4811 struct ring_buffer *buffer;
4812 struct print_entry *entry;
4813 unsigned long irq_flags;
4814 struct page *pages[2];
4824 if (tracing_disabled)
4827 if (!(trace_flags & TRACE_ITER_MARKERS))
4830 if (cnt > TRACE_BUF_SIZE)
4831 cnt = TRACE_BUF_SIZE;
4834 * Userspace is injecting traces into the kernel trace buffer.
4835 * We want to be as non intrusive as possible.
4836 * To do so, we do not want to allocate any special buffers
4837 * or take any locks, but instead write the userspace data
4838 * straight into the ring buffer.
4840 * First we need to pin the userspace buffer into memory,
4841 * which, most likely it is, because it just referenced it.
4842 * But there's no guarantee that it is. By using get_user_pages_fast()
4843 * and kmap_atomic/kunmap_atomic() we can get access to the
4844 * pages directly. We then write the data directly into the
4847 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4849 /* check if we cross pages */
4850 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4853 offset = addr & (PAGE_SIZE - 1);
4856 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4857 if (ret < nr_pages) {
4859 put_page(pages[ret]);
4864 for (i = 0; i < nr_pages; i++)
4865 map_page[i] = kmap_atomic(pages[i]);
4867 local_save_flags(irq_flags);
4868 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4869 buffer = tr->trace_buffer.buffer;
4870 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4871 irq_flags, preempt_count());
4873 /* Ring buffer disabled, return as if not open for write */
4878 entry = ring_buffer_event_data(event);
4879 entry->ip = _THIS_IP_;
4881 if (nr_pages == 2) {
4882 len = PAGE_SIZE - offset;
4883 memcpy(&entry->buf, map_page[0] + offset, len);
4884 memcpy(&entry->buf[len], map_page[1], cnt - len);
4886 memcpy(&entry->buf, map_page[0] + offset, cnt);
4888 if (entry->buf[cnt - 1] != '\n') {
4889 entry->buf[cnt] = '\n';
4890 entry->buf[cnt + 1] = '\0';
4892 entry->buf[cnt] = '\0';
4894 __buffer_unlock_commit(buffer, event);
4901 for (i = nr_pages - 1; i >= 0; i--) {
4902 kunmap_atomic(map_page[i]);
4909 static int tracing_clock_show(struct seq_file *m, void *v)
4911 struct trace_array *tr = m->private;
4914 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4916 "%s%s%s%s", i ? " " : "",
4917 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4918 i == tr->clock_id ? "]" : "");
4924 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4928 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4929 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4932 if (i == ARRAY_SIZE(trace_clocks))
4935 mutex_lock(&trace_types_lock);
4939 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4942 * New clock may not be consistent with the previous clock.
4943 * Reset the buffer so that it doesn't have incomparable timestamps.
4945 tracing_reset_online_cpus(&tr->trace_buffer);
4947 #ifdef CONFIG_TRACER_MAX_TRACE
4948 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4949 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4950 tracing_reset_online_cpus(&tr->max_buffer);
4953 mutex_unlock(&trace_types_lock);
4958 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4959 size_t cnt, loff_t *fpos)
4961 struct seq_file *m = filp->private_data;
4962 struct trace_array *tr = m->private;
4964 const char *clockstr;
4967 if (cnt >= sizeof(buf))
4970 if (copy_from_user(&buf, ubuf, cnt))
4975 clockstr = strstrip(buf);
4977 ret = tracing_set_clock(tr, clockstr);
4986 static int tracing_clock_open(struct inode *inode, struct file *file)
4988 struct trace_array *tr = inode->i_private;
4991 if (tracing_disabled)
4994 if (trace_array_get(tr))
4997 ret = single_open(file, tracing_clock_show, inode->i_private);
4999 trace_array_put(tr);
5004 struct ftrace_buffer_info {
5005 struct trace_iterator iter;
5010 #ifdef CONFIG_TRACER_SNAPSHOT
5011 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5013 struct trace_array *tr = inode->i_private;
5014 struct trace_iterator *iter;
5018 if (trace_array_get(tr) < 0)
5021 if (file->f_mode & FMODE_READ) {
5022 iter = __tracing_open(inode, file, true);
5024 ret = PTR_ERR(iter);
5026 /* Writes still need the seq_file to hold the private data */
5028 m = kzalloc(sizeof(*m), GFP_KERNEL);
5031 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5039 iter->trace_buffer = &tr->max_buffer;
5040 iter->cpu_file = tracing_get_cpu(inode);
5042 file->private_data = m;
5046 trace_array_put(tr);
5052 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5055 struct seq_file *m = filp->private_data;
5056 struct trace_iterator *iter = m->private;
5057 struct trace_array *tr = iter->tr;
5061 ret = tracing_update_buffers();
5065 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5069 mutex_lock(&trace_types_lock);
5071 if (tr->current_trace->use_max_tr) {
5078 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5082 if (tr->allocated_snapshot)
5086 /* Only allow per-cpu swap if the ring buffer supports it */
5087 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5088 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5093 if (!tr->allocated_snapshot) {
5094 ret = alloc_snapshot(tr);
5098 local_irq_disable();
5099 /* Now, we're going to swap */
5100 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5101 update_max_tr(tr, current, smp_processor_id());
5103 update_max_tr_single(tr, current, iter->cpu_file);
5107 if (tr->allocated_snapshot) {
5108 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5109 tracing_reset_online_cpus(&tr->max_buffer);
5111 tracing_reset(&tr->max_buffer, iter->cpu_file);
5121 mutex_unlock(&trace_types_lock);
5125 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5127 struct seq_file *m = file->private_data;
5130 ret = tracing_release(inode, file);
5132 if (file->f_mode & FMODE_READ)
5135 /* If write only, the seq_file is just a stub */
5143 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5144 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5145 size_t count, loff_t *ppos);
5146 static int tracing_buffers_release(struct inode *inode, struct file *file);
5147 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5148 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5150 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5152 struct ftrace_buffer_info *info;
5155 ret = tracing_buffers_open(inode, filp);
5159 info = filp->private_data;
5161 if (info->iter.trace->use_max_tr) {
5162 tracing_buffers_release(inode, filp);
5166 info->iter.snapshot = true;
5167 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5172 #endif /* CONFIG_TRACER_SNAPSHOT */
5175 static const struct file_operations tracing_max_lat_fops = {
5176 .open = tracing_open_generic,
5177 .read = tracing_max_lat_read,
5178 .write = tracing_max_lat_write,
5179 .llseek = generic_file_llseek,
5182 static const struct file_operations set_tracer_fops = {
5183 .open = tracing_open_generic,
5184 .read = tracing_set_trace_read,
5185 .write = tracing_set_trace_write,
5186 .llseek = generic_file_llseek,
5189 static const struct file_operations tracing_pipe_fops = {
5190 .open = tracing_open_pipe,
5191 .poll = tracing_poll_pipe,
5192 .read = tracing_read_pipe,
5193 .splice_read = tracing_splice_read_pipe,
5194 .release = tracing_release_pipe,
5195 .llseek = no_llseek,
5198 static const struct file_operations tracing_entries_fops = {
5199 .open = tracing_open_generic_tr,
5200 .read = tracing_entries_read,
5201 .write = tracing_entries_write,
5202 .llseek = generic_file_llseek,
5203 .release = tracing_release_generic_tr,
5206 static const struct file_operations tracing_total_entries_fops = {
5207 .open = tracing_open_generic_tr,
5208 .read = tracing_total_entries_read,
5209 .llseek = generic_file_llseek,
5210 .release = tracing_release_generic_tr,
5213 static const struct file_operations tracing_free_buffer_fops = {
5214 .open = tracing_open_generic_tr,
5215 .write = tracing_free_buffer_write,
5216 .release = tracing_free_buffer_release,
5219 static const struct file_operations tracing_mark_fops = {
5220 .open = tracing_open_generic_tr,
5221 .write = tracing_mark_write,
5222 .llseek = generic_file_llseek,
5223 .release = tracing_release_generic_tr,
5226 static const struct file_operations trace_clock_fops = {
5227 .open = tracing_clock_open,
5229 .llseek = seq_lseek,
5230 .release = tracing_single_release_tr,
5231 .write = tracing_clock_write,
5234 #ifdef CONFIG_TRACER_SNAPSHOT
5235 static const struct file_operations snapshot_fops = {
5236 .open = tracing_snapshot_open,
5238 .write = tracing_snapshot_write,
5239 .llseek = tracing_lseek,
5240 .release = tracing_snapshot_release,
5243 static const struct file_operations snapshot_raw_fops = {
5244 .open = snapshot_raw_open,
5245 .read = tracing_buffers_read,
5246 .release = tracing_buffers_release,
5247 .splice_read = tracing_buffers_splice_read,
5248 .llseek = no_llseek,
5251 #endif /* CONFIG_TRACER_SNAPSHOT */
5253 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5255 struct trace_array *tr = inode->i_private;
5256 struct ftrace_buffer_info *info;
5259 if (tracing_disabled)
5262 if (trace_array_get(tr) < 0)
5265 info = kzalloc(sizeof(*info), GFP_KERNEL);
5267 trace_array_put(tr);
5271 mutex_lock(&trace_types_lock);
5274 info->iter.cpu_file = tracing_get_cpu(inode);
5275 info->iter.trace = tr->current_trace;
5276 info->iter.trace_buffer = &tr->trace_buffer;
5278 /* Force reading ring buffer for first read */
5279 info->read = (unsigned int)-1;
5281 filp->private_data = info;
5283 mutex_unlock(&trace_types_lock);
5285 ret = nonseekable_open(inode, filp);
5287 trace_array_put(tr);
5293 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5295 struct ftrace_buffer_info *info = filp->private_data;
5296 struct trace_iterator *iter = &info->iter;
5298 return trace_poll(iter, filp, poll_table);
5302 tracing_buffers_read(struct file *filp, char __user *ubuf,
5303 size_t count, loff_t *ppos)
5305 struct ftrace_buffer_info *info = filp->private_data;
5306 struct trace_iterator *iter = &info->iter;
5313 mutex_lock(&trace_types_lock);
5315 #ifdef CONFIG_TRACER_MAX_TRACE
5316 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5323 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5329 /* Do we have previous read data to read? */
5330 if (info->read < PAGE_SIZE)
5334 trace_access_lock(iter->cpu_file);
5335 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5339 trace_access_unlock(iter->cpu_file);
5342 if (trace_empty(iter)) {
5343 if ((filp->f_flags & O_NONBLOCK)) {
5347 mutex_unlock(&trace_types_lock);
5348 ret = wait_on_pipe(iter, false);
5349 mutex_lock(&trace_types_lock);
5362 size = PAGE_SIZE - info->read;
5366 ret = copy_to_user(ubuf, info->spare + info->read, size);
5377 mutex_unlock(&trace_types_lock);
5382 static int tracing_buffers_release(struct inode *inode, struct file *file)
5384 struct ftrace_buffer_info *info = file->private_data;
5385 struct trace_iterator *iter = &info->iter;
5387 mutex_lock(&trace_types_lock);
5389 __trace_array_put(iter->tr);
5392 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5395 mutex_unlock(&trace_types_lock);
5401 struct ring_buffer *buffer;
5406 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5407 struct pipe_buffer *buf)
5409 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5414 ring_buffer_free_read_page(ref->buffer, ref->page);
5419 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5420 struct pipe_buffer *buf)
5422 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5427 /* Pipe buffer operations for a buffer. */
5428 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5430 .confirm = generic_pipe_buf_confirm,
5431 .release = buffer_pipe_buf_release,
5432 .steal = generic_pipe_buf_steal,
5433 .get = buffer_pipe_buf_get,
5437 * Callback from splice_to_pipe(), if we need to release some pages
5438 * at the end of the spd in case we error'ed out in filling the pipe.
5440 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5442 struct buffer_ref *ref =
5443 (struct buffer_ref *)spd->partial[i].private;
5448 ring_buffer_free_read_page(ref->buffer, ref->page);
5450 spd->partial[i].private = 0;
5454 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5455 struct pipe_inode_info *pipe, size_t len,
5458 struct ftrace_buffer_info *info = file->private_data;
5459 struct trace_iterator *iter = &info->iter;
5460 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5461 struct page *pages_def[PIPE_DEF_BUFFERS];
5462 struct splice_pipe_desc spd = {
5464 .partial = partial_def,
5465 .nr_pages_max = PIPE_DEF_BUFFERS,
5467 .ops = &buffer_pipe_buf_ops,
5468 .spd_release = buffer_spd_release,
5470 struct buffer_ref *ref;
5471 int entries, size, i;
5474 mutex_lock(&trace_types_lock);
5476 #ifdef CONFIG_TRACER_MAX_TRACE
5477 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5483 if (splice_grow_spd(pipe, &spd)) {
5488 if (*ppos & (PAGE_SIZE - 1)) {
5493 if (len & (PAGE_SIZE - 1)) {
5494 if (len < PAGE_SIZE) {
5502 trace_access_lock(iter->cpu_file);
5503 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5505 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5509 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5514 ref->buffer = iter->trace_buffer->buffer;
5515 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5521 r = ring_buffer_read_page(ref->buffer, &ref->page,
5522 len, iter->cpu_file, 1);
5524 ring_buffer_free_read_page(ref->buffer, ref->page);
5530 * zero out any left over data, this is going to
5533 size = ring_buffer_page_len(ref->page);
5534 if (size < PAGE_SIZE)
5535 memset(ref->page + size, 0, PAGE_SIZE - size);
5537 page = virt_to_page(ref->page);
5539 spd.pages[i] = page;
5540 spd.partial[i].len = PAGE_SIZE;
5541 spd.partial[i].offset = 0;
5542 spd.partial[i].private = (unsigned long)ref;
5546 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5549 trace_access_unlock(iter->cpu_file);
5552 /* did we read anything? */
5553 if (!spd.nr_pages) {
5554 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5558 mutex_unlock(&trace_types_lock);
5559 ret = wait_on_pipe(iter, true);
5560 mutex_lock(&trace_types_lock);
5567 ret = splice_to_pipe(pipe, &spd);
5568 splice_shrink_spd(&spd);
5570 mutex_unlock(&trace_types_lock);
5575 static const struct file_operations tracing_buffers_fops = {
5576 .open = tracing_buffers_open,
5577 .read = tracing_buffers_read,
5578 .poll = tracing_buffers_poll,
5579 .release = tracing_buffers_release,
5580 .splice_read = tracing_buffers_splice_read,
5581 .llseek = no_llseek,
5585 tracing_stats_read(struct file *filp, char __user *ubuf,
5586 size_t count, loff_t *ppos)
5588 struct inode *inode = file_inode(filp);
5589 struct trace_array *tr = inode->i_private;
5590 struct trace_buffer *trace_buf = &tr->trace_buffer;
5591 int cpu = tracing_get_cpu(inode);
5592 struct trace_seq *s;
5594 unsigned long long t;
5595 unsigned long usec_rem;
5597 s = kmalloc(sizeof(*s), GFP_KERNEL);
5603 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5604 trace_seq_printf(s, "entries: %ld\n", cnt);
5606 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5607 trace_seq_printf(s, "overrun: %ld\n", cnt);
5609 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5610 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5612 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5613 trace_seq_printf(s, "bytes: %ld\n", cnt);
5615 if (trace_clocks[tr->clock_id].in_ns) {
5616 /* local or global for trace_clock */
5617 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5618 usec_rem = do_div(t, USEC_PER_SEC);
5619 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5622 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5623 usec_rem = do_div(t, USEC_PER_SEC);
5624 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5626 /* counter or tsc mode for trace_clock */
5627 trace_seq_printf(s, "oldest event ts: %llu\n",
5628 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5630 trace_seq_printf(s, "now ts: %llu\n",
5631 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5634 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5635 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5637 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5638 trace_seq_printf(s, "read events: %ld\n", cnt);
5640 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5647 static const struct file_operations tracing_stats_fops = {
5648 .open = tracing_open_generic_tr,
5649 .read = tracing_stats_read,
5650 .llseek = generic_file_llseek,
5651 .release = tracing_release_generic_tr,
5654 #ifdef CONFIG_DYNAMIC_FTRACE
5656 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5662 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5663 size_t cnt, loff_t *ppos)
5665 static char ftrace_dyn_info_buffer[1024];
5666 static DEFINE_MUTEX(dyn_info_mutex);
5667 unsigned long *p = filp->private_data;
5668 char *buf = ftrace_dyn_info_buffer;
5669 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5672 mutex_lock(&dyn_info_mutex);
5673 r = sprintf(buf, "%ld ", *p);
5675 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5678 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5680 mutex_unlock(&dyn_info_mutex);
5685 static const struct file_operations tracing_dyn_info_fops = {
5686 .open = tracing_open_generic,
5687 .read = tracing_read_dyn_info,
5688 .llseek = generic_file_llseek,
5690 #endif /* CONFIG_DYNAMIC_FTRACE */
5692 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5694 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5700 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5702 unsigned long *count = (long *)data;
5714 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5715 struct ftrace_probe_ops *ops, void *data)
5717 long count = (long)data;
5719 seq_printf(m, "%ps:", (void *)ip);
5721 seq_printf(m, "snapshot");
5724 seq_printf(m, ":unlimited\n");
5726 seq_printf(m, ":count=%ld\n", count);
5731 static struct ftrace_probe_ops snapshot_probe_ops = {
5732 .func = ftrace_snapshot,
5733 .print = ftrace_snapshot_print,
5736 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5737 .func = ftrace_count_snapshot,
5738 .print = ftrace_snapshot_print,
5742 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5743 char *glob, char *cmd, char *param, int enable)
5745 struct ftrace_probe_ops *ops;
5746 void *count = (void *)-1;
5750 /* hash funcs only work with set_ftrace_filter */
5754 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5756 if (glob[0] == '!') {
5757 unregister_ftrace_function_probe_func(glob+1, ops);
5764 number = strsep(¶m, ":");
5766 if (!strlen(number))
5770 * We use the callback data field (which is a pointer)
5773 ret = kstrtoul(number, 0, (unsigned long *)&count);
5778 ret = register_ftrace_function_probe(glob, ops, count);
5781 alloc_snapshot(&global_trace);
5783 return ret < 0 ? ret : 0;
5786 static struct ftrace_func_command ftrace_snapshot_cmd = {
5788 .func = ftrace_trace_snapshot_callback,
5791 static __init int register_snapshot_cmd(void)
5793 return register_ftrace_command(&ftrace_snapshot_cmd);
5796 static inline __init int register_snapshot_cmd(void) { return 0; }
5797 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5799 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5804 if (!debugfs_initialized())
5807 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5808 tr->dir = debugfs_create_dir("tracing", NULL);
5811 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5816 struct dentry *tracing_init_dentry(void)
5818 return tracing_init_dentry_tr(&global_trace);
5821 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5823 struct dentry *d_tracer;
5826 return tr->percpu_dir;
5828 d_tracer = tracing_init_dentry_tr(tr);
5832 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5834 WARN_ONCE(!tr->percpu_dir,
5835 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5837 return tr->percpu_dir;
5840 static struct dentry *
5841 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5842 void *data, long cpu, const struct file_operations *fops)
5844 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5846 if (ret) /* See tracing_get_cpu() */
5847 ret->d_inode->i_cdev = (void *)(cpu + 1);
5852 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5854 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5855 struct dentry *d_cpu;
5856 char cpu_dir[30]; /* 30 characters should be more than enough */
5861 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5862 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5864 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5868 /* per cpu trace_pipe */
5869 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5870 tr, cpu, &tracing_pipe_fops);
5873 trace_create_cpu_file("trace", 0644, d_cpu,
5874 tr, cpu, &tracing_fops);
5876 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5877 tr, cpu, &tracing_buffers_fops);
5879 trace_create_cpu_file("stats", 0444, d_cpu,
5880 tr, cpu, &tracing_stats_fops);
5882 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5883 tr, cpu, &tracing_entries_fops);
5885 #ifdef CONFIG_TRACER_SNAPSHOT
5886 trace_create_cpu_file("snapshot", 0644, d_cpu,
5887 tr, cpu, &snapshot_fops);
5889 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5890 tr, cpu, &snapshot_raw_fops);
5894 #ifdef CONFIG_FTRACE_SELFTEST
5895 /* Let selftest have access to static functions in this file */
5896 #include "trace_selftest.c"
5899 struct trace_option_dentry {
5900 struct tracer_opt *opt;
5901 struct tracer_flags *flags;
5902 struct trace_array *tr;
5903 struct dentry *entry;
5907 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5910 struct trace_option_dentry *topt = filp->private_data;
5913 if (topt->flags->val & topt->opt->bit)
5918 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5922 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5925 struct trace_option_dentry *topt = filp->private_data;
5929 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5933 if (val != 0 && val != 1)
5936 if (!!(topt->flags->val & topt->opt->bit) != val) {
5937 mutex_lock(&trace_types_lock);
5938 ret = __set_tracer_option(topt->tr, topt->flags,
5940 mutex_unlock(&trace_types_lock);
5951 static const struct file_operations trace_options_fops = {
5952 .open = tracing_open_generic,
5953 .read = trace_options_read,
5954 .write = trace_options_write,
5955 .llseek = generic_file_llseek,
5959 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5962 long index = (long)filp->private_data;
5965 if (trace_flags & (1 << index))
5970 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5974 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5977 struct trace_array *tr = &global_trace;
5978 long index = (long)filp->private_data;
5982 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5986 if (val != 0 && val != 1)
5989 mutex_lock(&trace_types_lock);
5990 ret = set_tracer_flag(tr, 1 << index, val);
5991 mutex_unlock(&trace_types_lock);
6001 static const struct file_operations trace_options_core_fops = {
6002 .open = tracing_open_generic,
6003 .read = trace_options_core_read,
6004 .write = trace_options_core_write,
6005 .llseek = generic_file_llseek,
6008 struct dentry *trace_create_file(const char *name,
6010 struct dentry *parent,
6012 const struct file_operations *fops)
6016 ret = debugfs_create_file(name, mode, parent, data, fops);
6018 pr_warning("Could not create debugfs '%s' entry\n", name);
6024 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6026 struct dentry *d_tracer;
6031 d_tracer = tracing_init_dentry_tr(tr);
6035 tr->options = debugfs_create_dir("options", d_tracer);
6037 pr_warning("Could not create debugfs directory 'options'\n");
6045 create_trace_option_file(struct trace_array *tr,
6046 struct trace_option_dentry *topt,
6047 struct tracer_flags *flags,
6048 struct tracer_opt *opt)
6050 struct dentry *t_options;
6052 t_options = trace_options_init_dentry(tr);
6056 topt->flags = flags;
6060 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6061 &trace_options_fops);
6065 static struct trace_option_dentry *
6066 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6068 struct trace_option_dentry *topts;
6069 struct tracer_flags *flags;
6070 struct tracer_opt *opts;
6076 flags = tracer->flags;
6078 if (!flags || !flags->opts)
6083 for (cnt = 0; opts[cnt].name; cnt++)
6086 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6090 for (cnt = 0; opts[cnt].name; cnt++)
6091 create_trace_option_file(tr, &topts[cnt], flags,
6098 destroy_trace_option_files(struct trace_option_dentry *topts)
6105 for (cnt = 0; topts[cnt].opt; cnt++) {
6106 if (topts[cnt].entry)
6107 debugfs_remove(topts[cnt].entry);
6113 static struct dentry *
6114 create_trace_option_core_file(struct trace_array *tr,
6115 const char *option, long index)
6117 struct dentry *t_options;
6119 t_options = trace_options_init_dentry(tr);
6123 return trace_create_file(option, 0644, t_options, (void *)index,
6124 &trace_options_core_fops);
6127 static __init void create_trace_options_dir(struct trace_array *tr)
6129 struct dentry *t_options;
6132 t_options = trace_options_init_dentry(tr);
6136 for (i = 0; trace_options[i]; i++)
6137 create_trace_option_core_file(tr, trace_options[i], i);
6141 rb_simple_read(struct file *filp, char __user *ubuf,
6142 size_t cnt, loff_t *ppos)
6144 struct trace_array *tr = filp->private_data;
6148 r = tracer_tracing_is_on(tr);
6149 r = sprintf(buf, "%d\n", r);
6151 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6155 rb_simple_write(struct file *filp, const char __user *ubuf,
6156 size_t cnt, loff_t *ppos)
6158 struct trace_array *tr = filp->private_data;
6159 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6163 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6168 mutex_lock(&trace_types_lock);
6170 tracer_tracing_on(tr);
6171 if (tr->current_trace->start)
6172 tr->current_trace->start(tr);
6174 tracer_tracing_off(tr);
6175 if (tr->current_trace->stop)
6176 tr->current_trace->stop(tr);
6178 mutex_unlock(&trace_types_lock);
6186 static const struct file_operations rb_simple_fops = {
6187 .open = tracing_open_generic_tr,
6188 .read = rb_simple_read,
6189 .write = rb_simple_write,
6190 .release = tracing_release_generic_tr,
6191 .llseek = default_llseek,
6194 struct dentry *trace_instance_dir;
6197 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6200 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6202 enum ring_buffer_flags rb_flags;
6204 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6208 buf->buffer = ring_buffer_alloc(size, rb_flags);
6212 buf->data = alloc_percpu(struct trace_array_cpu);
6214 ring_buffer_free(buf->buffer);
6218 /* Allocate the first page for all buffers */
6219 set_buffer_entries(&tr->trace_buffer,
6220 ring_buffer_size(tr->trace_buffer.buffer, 0));
6225 static int allocate_trace_buffers(struct trace_array *tr, int size)
6229 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6233 #ifdef CONFIG_TRACER_MAX_TRACE
6234 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6235 allocate_snapshot ? size : 1);
6237 ring_buffer_free(tr->trace_buffer.buffer);
6238 free_percpu(tr->trace_buffer.data);
6241 tr->allocated_snapshot = allocate_snapshot;
6244 * Only the top level trace array gets its snapshot allocated
6245 * from the kernel command line.
6247 allocate_snapshot = false;
6252 static void free_trace_buffer(struct trace_buffer *buf)
6255 ring_buffer_free(buf->buffer);
6257 free_percpu(buf->data);
6262 static void free_trace_buffers(struct trace_array *tr)
6267 free_trace_buffer(&tr->trace_buffer);
6269 #ifdef CONFIG_TRACER_MAX_TRACE
6270 free_trace_buffer(&tr->max_buffer);
6274 static int new_instance_create(const char *name)
6276 struct trace_array *tr;
6279 mutex_lock(&trace_types_lock);
6282 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6283 if (tr->name && strcmp(tr->name, name) == 0)
6288 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6292 tr->name = kstrdup(name, GFP_KERNEL);
6296 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6299 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6301 raw_spin_lock_init(&tr->start_lock);
6303 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6305 tr->current_trace = &nop_trace;
6307 INIT_LIST_HEAD(&tr->systems);
6308 INIT_LIST_HEAD(&tr->events);
6310 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6313 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6317 ret = event_trace_add_tracer(tr->dir, tr);
6319 debugfs_remove_recursive(tr->dir);
6323 init_tracer_debugfs(tr, tr->dir);
6325 list_add(&tr->list, &ftrace_trace_arrays);
6327 mutex_unlock(&trace_types_lock);
6332 free_trace_buffers(tr);
6333 free_cpumask_var(tr->tracing_cpumask);
6338 mutex_unlock(&trace_types_lock);
6344 static int instance_delete(const char *name)
6346 struct trace_array *tr;
6350 mutex_lock(&trace_types_lock);
6353 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6354 if (tr->name && strcmp(tr->name, name) == 0) {
6366 list_del(&tr->list);
6368 tracing_set_nop(tr);
6369 event_trace_del_tracer(tr);
6370 ftrace_destroy_function_files(tr);
6371 debugfs_remove_recursive(tr->dir);
6372 free_trace_buffers(tr);
6380 mutex_unlock(&trace_types_lock);
6385 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6387 struct dentry *parent;
6390 /* Paranoid: Make sure the parent is the "instances" directory */
6391 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
6392 if (WARN_ON_ONCE(parent != trace_instance_dir))
6396 * The inode mutex is locked, but debugfs_create_dir() will also
6397 * take the mutex. As the instances directory can not be destroyed
6398 * or changed in any other way, it is safe to unlock it, and
6399 * let the dentry try. If two users try to make the same dir at
6400 * the same time, then the new_instance_create() will determine the
6403 mutex_unlock(&inode->i_mutex);
6405 ret = new_instance_create(dentry->d_iname);
6407 mutex_lock(&inode->i_mutex);
6412 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6414 struct dentry *parent;
6417 /* Paranoid: Make sure the parent is the "instances" directory */
6418 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
6419 if (WARN_ON_ONCE(parent != trace_instance_dir))
6422 /* The caller did a dget() on dentry */
6423 mutex_unlock(&dentry->d_inode->i_mutex);
6426 * The inode mutex is locked, but debugfs_create_dir() will also
6427 * take the mutex. As the instances directory can not be destroyed
6428 * or changed in any other way, it is safe to unlock it, and
6429 * let the dentry try. If two users try to make the same dir at
6430 * the same time, then the instance_delete() will determine the
6433 mutex_unlock(&inode->i_mutex);
6435 ret = instance_delete(dentry->d_iname);
6437 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6438 mutex_lock(&dentry->d_inode->i_mutex);
6443 static const struct inode_operations instance_dir_inode_operations = {
6444 .lookup = simple_lookup,
6445 .mkdir = instance_mkdir,
6446 .rmdir = instance_rmdir,
6449 static __init void create_trace_instances(struct dentry *d_tracer)
6451 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6452 if (WARN_ON(!trace_instance_dir))
6455 /* Hijack the dir inode operations, to allow mkdir */
6456 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6460 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6464 trace_create_file("available_tracers", 0444, d_tracer,
6465 tr, &show_traces_fops);
6467 trace_create_file("current_tracer", 0644, d_tracer,
6468 tr, &set_tracer_fops);
6470 trace_create_file("tracing_cpumask", 0644, d_tracer,
6471 tr, &tracing_cpumask_fops);
6473 trace_create_file("trace_options", 0644, d_tracer,
6474 tr, &tracing_iter_fops);
6476 trace_create_file("trace", 0644, d_tracer,
6479 trace_create_file("trace_pipe", 0444, d_tracer,
6480 tr, &tracing_pipe_fops);
6482 trace_create_file("buffer_size_kb", 0644, d_tracer,
6483 tr, &tracing_entries_fops);
6485 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6486 tr, &tracing_total_entries_fops);
6488 trace_create_file("free_buffer", 0200, d_tracer,
6489 tr, &tracing_free_buffer_fops);
6491 trace_create_file("trace_marker", 0220, d_tracer,
6492 tr, &tracing_mark_fops);
6494 trace_create_file("trace_clock", 0644, d_tracer, tr,
6497 trace_create_file("tracing_on", 0644, d_tracer,
6498 tr, &rb_simple_fops);
6500 #ifdef CONFIG_TRACER_MAX_TRACE
6501 trace_create_file("tracing_max_latency", 0644, d_tracer,
6502 &tr->max_latency, &tracing_max_lat_fops);
6505 if (ftrace_create_function_files(tr, d_tracer))
6506 WARN(1, "Could not allocate function filter files");
6508 #ifdef CONFIG_TRACER_SNAPSHOT
6509 trace_create_file("snapshot", 0644, d_tracer,
6510 tr, &snapshot_fops);
6513 for_each_tracing_cpu(cpu)
6514 tracing_init_debugfs_percpu(tr, cpu);
6518 static __init int tracer_init_debugfs(void)
6520 struct dentry *d_tracer;
6522 trace_access_lock_init();
6524 d_tracer = tracing_init_dentry();
6528 init_tracer_debugfs(&global_trace, d_tracer);
6530 trace_create_file("tracing_thresh", 0644, d_tracer,
6531 &tracing_thresh, &tracing_max_lat_fops);
6533 trace_create_file("README", 0444, d_tracer,
6534 NULL, &tracing_readme_fops);
6536 trace_create_file("saved_cmdlines", 0444, d_tracer,
6537 NULL, &tracing_saved_cmdlines_fops);
6539 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6540 NULL, &tracing_saved_cmdlines_size_fops);
6542 #ifdef CONFIG_DYNAMIC_FTRACE
6543 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6544 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6547 create_trace_instances(d_tracer);
6549 create_trace_options_dir(&global_trace);
6554 static int trace_panic_handler(struct notifier_block *this,
6555 unsigned long event, void *unused)
6557 if (ftrace_dump_on_oops)
6558 ftrace_dump(ftrace_dump_on_oops);
6562 static struct notifier_block trace_panic_notifier = {
6563 .notifier_call = trace_panic_handler,
6565 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6568 static int trace_die_handler(struct notifier_block *self,
6574 if (ftrace_dump_on_oops)
6575 ftrace_dump(ftrace_dump_on_oops);
6583 static struct notifier_block trace_die_notifier = {
6584 .notifier_call = trace_die_handler,
6589 * printk is set to max of 1024, we really don't need it that big.
6590 * Nothing should be printing 1000 characters anyway.
6592 #define TRACE_MAX_PRINT 1000
6595 * Define here KERN_TRACE so that we have one place to modify
6596 * it if we decide to change what log level the ftrace dump
6599 #define KERN_TRACE KERN_EMERG
6602 trace_printk_seq(struct trace_seq *s)
6604 /* Probably should print a warning here. */
6605 if (s->len >= TRACE_MAX_PRINT)
6606 s->len = TRACE_MAX_PRINT;
6608 /* should be zero ended, but we are paranoid. */
6609 s->buffer[s->len] = 0;
6611 printk(KERN_TRACE "%s", s->buffer);
6616 void trace_init_global_iter(struct trace_iterator *iter)
6618 iter->tr = &global_trace;
6619 iter->trace = iter->tr->current_trace;
6620 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6621 iter->trace_buffer = &global_trace.trace_buffer;
6623 if (iter->trace && iter->trace->open)
6624 iter->trace->open(iter);
6626 /* Annotate start of buffers if we had overruns */
6627 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6628 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6630 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6631 if (trace_clocks[iter->tr->clock_id].in_ns)
6632 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6635 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6637 /* use static because iter can be a bit big for the stack */
6638 static struct trace_iterator iter;
6639 static atomic_t dump_running;
6640 unsigned int old_userobj;
6641 unsigned long flags;
6644 /* Only allow one dump user at a time. */
6645 if (atomic_inc_return(&dump_running) != 1) {
6646 atomic_dec(&dump_running);
6651 * Always turn off tracing when we dump.
6652 * We don't need to show trace output of what happens
6653 * between multiple crashes.
6655 * If the user does a sysrq-z, then they can re-enable
6656 * tracing with echo 1 > tracing_on.
6660 local_irq_save(flags);
6662 /* Simulate the iterator */
6663 trace_init_global_iter(&iter);
6665 for_each_tracing_cpu(cpu) {
6666 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6669 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6671 /* don't look at user memory in panic mode */
6672 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6674 switch (oops_dump_mode) {
6676 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6679 iter.cpu_file = raw_smp_processor_id();
6684 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6685 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6688 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6690 /* Did function tracer already get disabled? */
6691 if (ftrace_is_dead()) {
6692 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6693 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6697 * We need to stop all tracing on all CPUS to read the
6698 * the next buffer. This is a bit expensive, but is
6699 * not done often. We fill all what we can read,
6700 * and then release the locks again.
6703 while (!trace_empty(&iter)) {
6706 printk(KERN_TRACE "---------------------------------\n");
6710 /* reset all but tr, trace, and overruns */
6711 memset(&iter.seq, 0,
6712 sizeof(struct trace_iterator) -
6713 offsetof(struct trace_iterator, seq));
6714 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6717 if (trace_find_next_entry_inc(&iter) != NULL) {
6720 ret = print_trace_line(&iter);
6721 if (ret != TRACE_TYPE_NO_CONSUME)
6722 trace_consume(&iter);
6724 touch_nmi_watchdog();
6726 trace_printk_seq(&iter.seq);
6730 printk(KERN_TRACE " (ftrace buffer empty)\n");
6732 printk(KERN_TRACE "---------------------------------\n");
6735 trace_flags |= old_userobj;
6737 for_each_tracing_cpu(cpu) {
6738 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6740 atomic_dec(&dump_running);
6741 local_irq_restore(flags);
6743 EXPORT_SYMBOL_GPL(ftrace_dump);
6745 __init static int tracer_alloc_buffers(void)
6751 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6754 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6755 goto out_free_buffer_mask;
6757 /* Only allocate trace_printk buffers if a trace_printk exists */
6758 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6759 /* Must be called before global_trace.buffer is allocated */
6760 trace_printk_init_buffers();
6762 /* To save memory, keep the ring buffer size to its minimum */
6763 if (ring_buffer_expanded)
6764 ring_buf_size = trace_buf_size;
6768 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6769 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6771 raw_spin_lock_init(&global_trace.start_lock);
6773 /* Used for event triggers */
6774 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6776 goto out_free_cpumask;
6778 if (trace_create_savedcmd() < 0)
6779 goto out_free_temp_buffer;
6781 /* TODO: make the number of buffers hot pluggable with CPUS */
6782 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6783 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6785 goto out_free_savedcmd;
6788 if (global_trace.buffer_disabled)
6791 if (trace_boot_clock) {
6792 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6794 pr_warning("Trace clock %s not defined, going back to default\n",
6799 * register_tracer() might reference current_trace, so it
6800 * needs to be set before we register anything. This is
6801 * just a bootstrap of current_trace anyway.
6803 global_trace.current_trace = &nop_trace;
6805 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6807 ftrace_init_global_array_ops(&global_trace);
6809 register_tracer(&nop_trace);
6811 /* All seems OK, enable tracing */
6812 tracing_disabled = 0;
6814 atomic_notifier_chain_register(&panic_notifier_list,
6815 &trace_panic_notifier);
6817 register_die_notifier(&trace_die_notifier);
6819 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6821 INIT_LIST_HEAD(&global_trace.systems);
6822 INIT_LIST_HEAD(&global_trace.events);
6823 list_add(&global_trace.list, &ftrace_trace_arrays);
6825 while (trace_boot_options) {
6828 option = strsep(&trace_boot_options, ",");
6829 trace_set_options(&global_trace, option);
6832 register_snapshot_cmd();
6837 free_saved_cmdlines_buffer(savedcmd);
6838 out_free_temp_buffer:
6839 ring_buffer_free(temp_buffer);
6841 free_cpumask_var(global_trace.tracing_cpumask);
6842 out_free_buffer_mask:
6843 free_cpumask_var(tracing_buffer_mask);
6848 __init static int clear_boot_tracer(void)
6851 * The default tracer at boot buffer is an init section.
6852 * This function is called in lateinit. If we did not
6853 * find the boot tracer, then clear it out, to prevent
6854 * later registration from accessing the buffer that is
6855 * about to be freed.
6857 if (!default_bootup_tracer)
6860 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6861 default_bootup_tracer);
6862 default_bootup_tracer = NULL;
6867 early_initcall(tracer_alloc_buffers);
6868 fs_initcall(tracer_init_debugfs);
6869 late_initcall(clear_boot_tracer);