summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
Diffstat (limited to 'include')
-rw-r--r--include/linux/ftrace.h6
-rw-r--r--include/linux/ftrace_event.h111
-rw-r--r--include/linux/kernel.h70
-rw-r--r--include/linux/ring_buffer.h6
-rw-r--r--include/linux/trace_clock.h1
-rw-r--r--include/trace/ftrace.h49
6 files changed, 186 insertions, 57 deletions
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 52da2a25079..f83e17a40e8 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -261,8 +261,10 @@ struct ftrace_probe_ops {
void (*func)(unsigned long ip,
unsigned long parent_ip,
void **data);
- int (*callback)(unsigned long ip, void **data);
- void (*free)(void **data);
+ int (*init)(struct ftrace_probe_ops *ops,
+ unsigned long ip, void **data);
+ void (*free)(struct ftrace_probe_ops *ops,
+ unsigned long ip, void **data);
int (*print)(struct seq_file *m,
unsigned long ip,
struct ftrace_probe_ops *ops,
diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h
index 13a54d0bdfa..34e00fb49be 100644
--- a/include/linux/ftrace_event.h
+++ b/include/linux/ftrace_event.h
@@ -8,6 +8,7 @@
#include <linux/perf_event.h>
struct trace_array;
+struct trace_buffer;
struct tracer;
struct dentry;
@@ -38,6 +39,12 @@ const char *ftrace_print_symbols_seq_u64(struct trace_seq *p,
const char *ftrace_print_hex_seq(struct trace_seq *p,
const unsigned char *buf, int len);
+struct trace_iterator;
+struct trace_event;
+
+int ftrace_raw_output_prep(struct trace_iterator *iter,
+ struct trace_event *event);
+
/*
* The trace entry - the most basic unit of tracing. This is what
* is printed in the end as a single line in the trace output, such as:
@@ -61,6 +68,7 @@ struct trace_entry {
struct trace_iterator {
struct trace_array *tr;
struct tracer *trace;
+ struct trace_buffer *trace_buffer;
void *private;
int cpu_file;
struct mutex mutex;
@@ -95,8 +103,6 @@ enum trace_iter_flags {
};
-struct trace_event;
-
typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter,
int flags, struct trace_event *event);
@@ -128,6 +134,13 @@ enum print_line_t {
void tracing_generic_entry_update(struct trace_entry *entry,
unsigned long flags,
int pc);
+struct ftrace_event_file;
+
+struct ring_buffer_event *
+trace_event_buffer_lock_reserve(struct ring_buffer **current_buffer,
+ struct ftrace_event_file *ftrace_file,
+ int type, unsigned long len,
+ unsigned long flags, int pc);
struct ring_buffer_event *
trace_current_buffer_lock_reserve(struct ring_buffer **current_buffer,
int type, unsigned long len,
@@ -182,53 +195,49 @@ extern int ftrace_event_reg(struct ftrace_event_call *event,
enum trace_reg type, void *data);
enum {
- TRACE_EVENT_FL_ENABLED_BIT,
TRACE_EVENT_FL_FILTERED_BIT,
- TRACE_EVENT_FL_RECORDED_CMD_BIT,
TRACE_EVENT_FL_CAP_ANY_BIT,
TRACE_EVENT_FL_NO_SET_FILTER_BIT,
TRACE_EVENT_FL_IGNORE_ENABLE_BIT,
+ TRACE_EVENT_FL_WAS_ENABLED_BIT,
};
+/*
+ * Event flags:
+ * FILTERED - The event has a filter attached
+ * CAP_ANY - Any user can enable for perf
+ * NO_SET_FILTER - Set when filter has error and is to be ignored
+ * IGNORE_ENABLE - For ftrace internal events, do not enable with debugfs file
+ * WAS_ENABLED - Set and stays set when an event was ever enabled
+ * (used for module unloading, if a module event is enabled,
+ * it is best to clear the buffers that used it).
+ */
enum {
- TRACE_EVENT_FL_ENABLED = (1 << TRACE_EVENT_FL_ENABLED_BIT),
TRACE_EVENT_FL_FILTERED = (1 << TRACE_EVENT_FL_FILTERED_BIT),
- TRACE_EVENT_FL_RECORDED_CMD = (1 << TRACE_EVENT_FL_RECORDED_CMD_BIT),
TRACE_EVENT_FL_CAP_ANY = (1 << TRACE_EVENT_FL_CAP_ANY_BIT),
TRACE_EVENT_FL_NO_SET_FILTER = (1 << TRACE_EVENT_FL_NO_SET_FILTER_BIT),
TRACE_EVENT_FL_IGNORE_ENABLE = (1 << TRACE_EVENT_FL_IGNORE_ENABLE_BIT),
+ TRACE_EVENT_FL_WAS_ENABLED = (1 << TRACE_EVENT_FL_WAS_ENABLED_BIT),
};
struct ftrace_event_call {
struct list_head list;
struct ftrace_event_class *class;
char *name;
- struct dentry *dir;
struct trace_event event;
const char *print_fmt;
struct event_filter *filter;
+ struct list_head *files;
void *mod;
void *data;
-
/*
- * 32 bit flags:
- * bit 1: enabled
- * bit 2: filter_active
- * bit 3: enabled cmd record
- * bit 4: allow trace by non root (cap any)
- * bit 5: failed to apply filter
- * bit 6: ftrace internal event (do not enable)
- *
- * Changes to flags must hold the event_mutex.
- *
- * Note: Reads of flags do not hold the event_mutex since
- * they occur in critical sections. But the way flags
- * is currently used, these changes do no affect the code
- * except that when a change is made, it may have a slight
- * delay in propagating the changes to other CPUs due to
- * caching and such.
+ * bit 0: filter_active
+ * bit 1: allow trace by non root (cap any)
+ * bit 2: failed to apply filter
+ * bit 3: ftrace internal event (do not enable)
+ * bit 4: Event was enabled by module
*/
- unsigned int flags;
+ int flags; /* static flags of different events */
#ifdef CONFIG_PERF_EVENTS
int perf_refcount;
@@ -236,6 +245,56 @@ struct ftrace_event_call {
#endif
};
+struct trace_array;
+struct ftrace_subsystem_dir;
+
+enum {
+ FTRACE_EVENT_FL_ENABLED_BIT,
+ FTRACE_EVENT_FL_RECORDED_CMD_BIT,
+ FTRACE_EVENT_FL_SOFT_MODE_BIT,
+ FTRACE_EVENT_FL_SOFT_DISABLED_BIT,
+};
+
+/*
+ * Ftrace event file flags:
+ * ENABLED - The event is enabled
+ * RECORDED_CMD - The comms should be recorded at sched_switch
+ * SOFT_MODE - The event is enabled/disabled by SOFT_DISABLED
+ * SOFT_DISABLED - When set, do not trace the event (even though its
+ * tracepoint may be enabled)
+ */
+enum {
+ FTRACE_EVENT_FL_ENABLED = (1 << FTRACE_EVENT_FL_ENABLED_BIT),
+ FTRACE_EVENT_FL_RECORDED_CMD = (1 << FTRACE_EVENT_FL_RECORDED_CMD_BIT),
+ FTRACE_EVENT_FL_SOFT_MODE = (1 << FTRACE_EVENT_FL_SOFT_MODE_BIT),
+ FTRACE_EVENT_FL_SOFT_DISABLED = (1 << FTRACE_EVENT_FL_SOFT_DISABLED_BIT),
+};
+
+struct ftrace_event_file {
+ struct list_head list;
+ struct ftrace_event_call *event_call;
+ struct dentry *dir;
+ struct trace_array *tr;
+ struct ftrace_subsystem_dir *system;
+
+ /*
+ * 32 bit flags:
+ * bit 0: enabled
+ * bit 1: enabled cmd record
+ * bit 2: enable/disable with the soft disable bit
+ * bit 3: soft disabled
+ *
+ * Note: The bits must be set atomically to prevent races
+ * from other writers. Reads of flags do not need to be in
+ * sync as they occur in critical sections. But the way flags
+ * is currently used, these changes do not affect the code
+ * except that when a change is made, it may have a slight
+ * delay in propagating the changes to other CPUs due to
+ * caching and such. Which is mostly OK ;-)
+ */
+ unsigned long flags;
+};
+
#define __TRACE_EVENT_FLAGS(name, value) \
static int __init trace_init_flags_##name(void) \
{ \
@@ -274,7 +333,7 @@ extern int trace_define_field(struct ftrace_event_call *call, const char *type,
extern int trace_add_event_call(struct ftrace_event_call *call);
extern void trace_remove_event_call(struct ftrace_event_call *call);
-#define is_signed_type(type) (((type)(-1)) < (type)0)
+#define is_signed_type(type) (((type)(-1)) < (type)1)
int trace_set_clr_event(const char *system, const char *event, int set);
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 79fdd80a42d..2dac79c3919 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -486,6 +486,8 @@ enum ftrace_dump_mode {
void tracing_on(void);
void tracing_off(void);
int tracing_is_on(void);
+void tracing_snapshot(void);
+void tracing_snapshot_alloc(void);
extern void tracing_start(void);
extern void tracing_stop(void);
@@ -515,10 +517,32 @@ do { \
*
* This is intended as a debugging tool for the developer only.
* Please refrain from leaving trace_printks scattered around in
- * your code.
+ * your code. (Extra memory is used for special buffers that are
+ * allocated when trace_printk() is used)
+ *
+ * A little optization trick is done here. If there's only one
+ * argument, there's no need to scan the string for printf formats.
+ * The trace_puts() will suffice. But how can we take advantage of
+ * using trace_puts() when trace_printk() has only one argument?
+ * By stringifying the args and checking the size we can tell
+ * whether or not there are args. __stringify((__VA_ARGS__)) will
+ * turn into "()\0" with a size of 3 when there are no args, anything
+ * else will be bigger. All we need to do is define a string to this,
+ * and then take its size and compare to 3. If it's bigger, use
+ * do_trace_printk() otherwise, optimize it to trace_puts(). Then just
+ * let gcc optimize the rest.
*/
-#define trace_printk(fmt, args...) \
+#define trace_printk(fmt, ...) \
+do { \
+ char _______STR[] = __stringify((__VA_ARGS__)); \
+ if (sizeof(_______STR) > 3) \
+ do_trace_printk(fmt, ##__VA_ARGS__); \
+ else \
+ trace_puts(fmt); \
+} while (0)
+
+#define do_trace_printk(fmt, args...) \
do { \
static const char *trace_printk_fmt \
__attribute__((section("__trace_printk_fmt"))) = \
@@ -538,7 +562,45 @@ int __trace_bprintk(unsigned long ip, const char *fmt, ...);
extern __printf(2, 3)
int __trace_printk(unsigned long ip, const char *fmt, ...);
-extern void trace_dump_stack(void);
+/**
+ * trace_puts - write a string into the ftrace buffer
+ * @str: the string to record
+ *
+ * Note: __trace_bputs is an internal function for trace_puts and
+ * the @ip is passed in via the trace_puts macro.
+ *
+ * This is similar to trace_printk() but is made for those really fast
+ * paths that a developer wants the least amount of "Heisenbug" affects,
+ * where the processing of the print format is still too much.
+ *
+ * This function allows a kernel developer to debug fast path sections
+ * that printk is not appropriate for. By scattering in various
+ * printk like tracing in the code, a developer can quickly see
+ * where problems are occurring.
+ *
+ * This is intended as a debugging tool for the developer only.
+ * Please refrain from leaving trace_puts scattered around in
+ * your code. (Extra memory is used for special buffers that are
+ * allocated when trace_puts() is used)
+ *
+ * Returns: 0 if nothing was written, positive # if string was.
+ * (1 when __trace_bputs is used, strlen(str) when __trace_puts is used)
+ */
+
+extern int __trace_bputs(unsigned long ip, const char *str);
+extern int __trace_puts(unsigned long ip, const char *str, int size);
+#define trace_puts(str) ({ \
+ static const char *trace_printk_fmt \
+ __attribute__((section("__trace_printk_fmt"))) = \
+ __builtin_constant_p(str) ? str : NULL; \
+ \
+ if (__builtin_constant_p(str)) \
+ __trace_bputs(_THIS_IP_, trace_printk_fmt); \
+ else \
+ __trace_puts(_THIS_IP_, str, strlen(str)); \
+})
+
+extern void trace_dump_stack(int skip);
/*
* The double __builtin_constant_p is because gcc will give us an error
@@ -573,6 +635,8 @@ static inline void trace_dump_stack(void) { }
static inline void tracing_on(void) { }
static inline void tracing_off(void) { }
static inline int tracing_is_on(void) { return 0; }
+static inline void tracing_snapshot(void) { }
+static inline void tracing_snapshot_alloc(void) { }
static inline __printf(1, 2)
int trace_printk(const char *fmt, ...)
diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h
index 1342e69542f..d69cf637a15 100644
--- a/include/linux/ring_buffer.h
+++ b/include/linux/ring_buffer.h
@@ -4,6 +4,7 @@
#include <linux/kmemcheck.h>
#include <linux/mm.h>
#include <linux/seq_file.h>
+#include <linux/poll.h>
struct ring_buffer;
struct ring_buffer_iter;
@@ -96,6 +97,11 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k
__ring_buffer_alloc((size), (flags), &__key); \
})
+void ring_buffer_wait(struct ring_buffer *buffer, int cpu);
+int ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
+ struct file *filp, poll_table *poll_table);
+
+
#define RING_BUFFER_ALL_CPUS -1
void ring_buffer_free(struct ring_buffer *buffer);
diff --git a/include/linux/trace_clock.h b/include/linux/trace_clock.h
index d563f37e1a1..1d7ca273927 100644
--- a/include/linux/trace_clock.h
+++ b/include/linux/trace_clock.h
@@ -16,6 +16,7 @@
extern u64 notrace trace_clock_local(void);
extern u64 notrace trace_clock(void);
+extern u64 notrace trace_clock_jiffies(void);
extern u64 notrace trace_clock_global(void);
extern u64 notrace trace_clock_counter(void);
diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h
index 40dc5e8fe34..19edd7facaa 100644
--- a/include/trace/ftrace.h
+++ b/include/trace/ftrace.h
@@ -227,29 +227,18 @@ static notrace enum print_line_t \
ftrace_raw_output_##call(struct trace_iterator *iter, int flags, \
struct trace_event *trace_event) \
{ \
- struct ftrace_event_call *event; \
struct trace_seq *s = &iter->seq; \
+ struct trace_seq __maybe_unused *p = &iter->tmp_seq; \
struct ftrace_raw_##call *field; \
- struct trace_entry *entry; \
- struct trace_seq *p = &iter->tmp_seq; \
int ret; \
\
- event = container_of(trace_event, struct ftrace_event_call, \
- event); \
- \
- entry = iter->ent; \
- \
- if (entry->type != event->event.type) { \
- WARN_ON_ONCE(1); \
- return TRACE_TYPE_UNHANDLED; \
- } \
- \
- field = (typeof(field))entry; \
+ field = (typeof(field))iter->ent; \
\
- trace_seq_init(p); \
- ret = trace_seq_printf(s, "%s: ", event->name); \
+ ret = ftrace_raw_output_prep(iter, trace_event); \
if (ret) \
- ret = trace_seq_printf(s, print); \
+ return ret; \
+ \
+ ret = trace_seq_printf(s, print); \
if (!ret) \
return TRACE_TYPE_PARTIAL_LINE; \
\
@@ -335,7 +324,7 @@ static struct trace_event_functions ftrace_event_type_funcs_##call = { \
#undef DECLARE_EVENT_CLASS
#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \
-static int notrace \
+static int notrace __init \
ftrace_define_fields_##call(struct ftrace_event_call *event_call) \
{ \
struct ftrace_raw_##call field; \
@@ -414,7 +403,8 @@ static inline notrace int ftrace_get_offsets_##call( \
*
* static void ftrace_raw_event_<call>(void *__data, proto)
* {
- * struct ftrace_event_call *event_call = __data;
+ * struct ftrace_event_file *ftrace_file = __data;
+ * struct ftrace_event_call *event_call = ftrace_file->event_call;
* struct ftrace_data_offsets_<call> __maybe_unused __data_offsets;
* struct ring_buffer_event *event;
* struct ftrace_raw_<call> *entry; <-- defined in stage 1
@@ -423,12 +413,16 @@ static inline notrace int ftrace_get_offsets_##call( \
* int __data_size;
* int pc;
*
+ * if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT,
+ * &ftrace_file->flags))
+ * return;
+ *
* local_save_flags(irq_flags);
* pc = preempt_count();
*
* __data_size = ftrace_get_offsets_<call>(&__data_offsets, args);
*
- * event = trace_current_buffer_lock_reserve(&buffer,
+ * event = trace_event_buffer_lock_reserve(&buffer, ftrace_file,
* event_<call>->event.type,
* sizeof(*entry) + __data_size,
* irq_flags, pc);
@@ -440,7 +434,7 @@ static inline notrace int ftrace_get_offsets_##call( \
* __array macros.
*
* if (!filter_current_check_discard(buffer, event_call, entry, event))
- * trace_current_buffer_unlock_commit(buffer,
+ * trace_nowake_buffer_unlock_commit(buffer,
* event, irq_flags, pc);
* }
*
@@ -518,7 +512,8 @@ static inline notrace int ftrace_get_offsets_##call( \
static notrace void \
ftrace_raw_event_##call(void *__data, proto) \
{ \
- struct ftrace_event_call *event_call = __data; \
+ struct ftrace_event_file *ftrace_file = __data; \
+ struct ftrace_event_call *event_call = ftrace_file->event_call; \
struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
struct ring_buffer_event *event; \
struct ftrace_raw_##call *entry; \
@@ -527,12 +522,16 @@ ftrace_raw_event_##call(void *__data, proto) \
int __data_size; \
int pc; \
\
+ if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, \
+ &ftrace_file->flags)) \
+ return; \
+ \
local_save_flags(irq_flags); \
pc = preempt_count(); \
\
__data_size = ftrace_get_offsets_##call(&__data_offsets, args); \
\
- event = trace_current_buffer_lock_reserve(&buffer, \
+ event = trace_event_buffer_lock_reserve(&buffer, ftrace_file, \
event_call->event.type, \
sizeof(*entry) + __data_size, \
irq_flags, pc); \
@@ -581,7 +580,7 @@ static inline void ftrace_test_probe_##call(void) \
#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
_TRACE_PERF_PROTO(call, PARAMS(proto)); \
static const char print_fmt_##call[] = print; \
-static struct ftrace_event_class __used event_class_##call = { \
+static struct ftrace_event_class __used __refdata event_class_##call = { \
.system = __stringify(TRACE_SYSTEM), \
.define_fields = ftrace_define_fields_##call, \
.fields = LIST_HEAD_INIT(event_class_##call.fields),\
@@ -705,5 +704,3 @@ static inline void perf_test_probe_##call(void) \
#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
#endif /* CONFIG_PERF_EVENTS */
-#undef _TRACE_PROFILE_INIT
-
generated by cgit v1.2.3-70-g09d2 (git 2.51.2) at 2025-11-04 14:44:19 +0000