diff options
| -rw-r--r-- | include/linux/ftrace_event.h | 9 | ||||
| -rw-r--r-- | include/trace/ftrace.h | 48 | ||||
| -rw-r--r-- | kernel/trace/trace.h | 5 | ||||
| -rw-r--r-- | kernel/trace/trace_events_filter.c | 12 | ||||
| -rw-r--r-- | kernel/trace/trace_events_trigger.c | 170 | ||||
| -rw-r--r-- | kernel/trace/trace_syscalls.c | 46 |
6 files changed, 263 insertions, 27 deletions
diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h index 2f73c3988fc7..03d2db22ad0d 100644 --- a/include/linux/ftrace_event.h +++ b/include/linux/ftrace_event.h @@ -1,3 +1,4 @@ + #ifndef _LINUX_FTRACE_EVENT_H #define _LINUX_FTRACE_EVENT_H @@ -265,6 +266,7 @@ enum { FTRACE_EVENT_FL_SOFT_MODE_BIT, FTRACE_EVENT_FL_SOFT_DISABLED_BIT, FTRACE_EVENT_FL_TRIGGER_MODE_BIT, + FTRACE_EVENT_FL_TRIGGER_COND_BIT, }; /* @@ -277,6 +279,7 @@ enum { * SOFT_DISABLED - When set, do not trace the event (even though its * tracepoint may be enabled) * TRIGGER_MODE - When set, invoke the triggers associated with the event + * TRIGGER_COND - When set, one or more triggers has an associated filter */ enum { FTRACE_EVENT_FL_ENABLED = (1 << FTRACE_EVENT_FL_ENABLED_BIT), @@ -286,6 +289,7 @@ enum { FTRACE_EVENT_FL_SOFT_MODE = (1 << FTRACE_EVENT_FL_SOFT_MODE_BIT), FTRACE_EVENT_FL_SOFT_DISABLED = (1 << FTRACE_EVENT_FL_SOFT_DISABLED_BIT), FTRACE_EVENT_FL_TRIGGER_MODE = (1 << FTRACE_EVENT_FL_TRIGGER_MODE_BIT), + FTRACE_EVENT_FL_TRIGGER_COND = (1 << FTRACE_EVENT_FL_TRIGGER_COND_BIT), }; struct ftrace_event_file { @@ -361,7 +365,10 @@ extern int filter_check_discard(struct ftrace_event_file *file, void *rec, extern int call_filter_check_discard(struct ftrace_event_call *call, void *rec, struct ring_buffer *buffer, struct ring_buffer_event *event); -extern void event_triggers_call(struct ftrace_event_file *file); +extern enum event_trigger_type event_triggers_call(struct ftrace_event_file *file, + void *rec); +extern void event_triggers_post_call(struct ftrace_event_file *file, + enum event_trigger_type tt); enum { FILTER_OTHER = 0, diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h index 0a48bff964bd..0962968b8b37 100644 --- a/include/trace/ftrace.h +++ b/include/trace/ftrace.h @@ -418,6 +418,8 @@ static inline notrace int ftrace_get_offsets_##call( \ * struct ftrace_event_file *ftrace_file = __data; * struct ftrace_event_call *event_call = ftrace_file->event_call; * struct ftrace_data_offsets_<call> __maybe_unused __data_offsets; + * unsigned long eflags = ftrace_file->flags; + * enum event_trigger_type __tt = ETT_NONE; * struct ring_buffer_event *event; * struct ftrace_raw_<call> *entry; <-- defined in stage 1 * struct ring_buffer *buffer; @@ -425,9 +427,12 @@ static inline notrace int ftrace_get_offsets_##call( \ * int __data_size; * int pc; * - * if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, - * &ftrace_file->flags)) - * return; + * if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) { + * if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE) + * event_triggers_call(ftrace_file, NULL); + * if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED) + * return; + * } * * local_save_flags(irq_flags); * pc = preempt_count(); @@ -445,8 +450,17 @@ static inline notrace int ftrace_get_offsets_##call( \ * { <assign>; } <-- Here we assign the entries by the __field and * __array macros. * - * if (!filter_check_discard(ftrace_file, entry, buffer, event)) + * if (eflags & FTRACE_EVENT_FL_TRIGGER_COND) + * __tt = event_triggers_call(ftrace_file, entry); + * + * if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, + * &ftrace_file->flags)) + * ring_buffer_discard_commit(buffer, event); + * else if (!filter_check_discard(ftrace_file, entry, buffer, event)) * trace_buffer_unlock_commit(buffer, event, irq_flags, pc); + * + * if (__tt) + * event_triggers_post_call(ftrace_file, __tt); * } * * static struct trace_event ftrace_event_type_<call> = { @@ -532,6 +546,8 @@ ftrace_raw_event_##call(void *__data, proto) \ struct ftrace_event_file *ftrace_file = __data; \ struct ftrace_event_call *event_call = ftrace_file->event_call; \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ + unsigned long eflags = ftrace_file->flags; \ + enum event_trigger_type __tt = ETT_NONE; \ struct ring_buffer_event *event; \ struct ftrace_raw_##call *entry; \ struct ring_buffer *buffer; \ @@ -539,13 +555,12 @@ ftrace_raw_event_##call(void *__data, proto) \ int __data_size; \ int pc; \ \ - if (test_bit(FTRACE_EVENT_FL_TRIGGER_MODE_BIT, \ - &ftrace_file->flags)) \ - event_triggers_call(ftrace_file); \ - \ - if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, \ - &ftrace_file->flags)) \ - return; \ + if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) { \ + if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE) \ + event_triggers_call(ftrace_file, NULL); \ + if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED) \ + return; \ + } \ \ local_save_flags(irq_flags); \ pc = preempt_count(); \ @@ -564,8 +579,17 @@ ftrace_raw_event_##call(void *__data, proto) \ \ { assign; } \ \ - if (!filter_check_discard(ftrace_file, entry, buffer, event)) \ + if (eflags & FTRACE_EVENT_FL_TRIGGER_COND) \ + __tt = event_triggers_call(ftrace_file, entry); \ + \ + if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, \ + &ftrace_file->flags)) \ + ring_buffer_discard_commit(buffer, event); \ + else if (!filter_check_discard(ftrace_file, entry, buffer, event)) \ trace_buffer_unlock_commit(buffer, event, irq_flags, pc); \ + \ + if (__tt) \ + event_triggers_post_call(ftrace_file, __tt); \ } /* * The ftrace_test_probe is compiled out, it is only here as a build time check diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index ccbd8104cf99..433bfc5dd576 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1,3 +1,4 @@ + #ifndef _LINUX_KERNEL_TRACE_H #define _LINUX_KERNEL_TRACE_H @@ -1020,6 +1021,10 @@ extern int apply_subsystem_event_filter(struct ftrace_subsystem_dir *dir, extern void print_subsystem_event_filter(struct event_subsystem *system, struct trace_seq *s); extern int filter_assign_type(const char *type); +extern int create_event_filter(struct ftrace_event_call *call, + char *filter_str, bool set_str, + struct event_filter **filterp); +extern void free_event_filter(struct event_filter *filter); struct ftrace_event_field * trace_find_event_field(struct ftrace_event_call *call, char *name); diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 2468f56dc5db..8a8631926a07 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -799,6 +799,11 @@ static void __free_filter(struct event_filter *filter) kfree(filter); } +void free_event_filter(struct event_filter *filter) +{ + __free_filter(filter); +} + void destroy_call_preds(struct ftrace_event_call *call) { __free_filter(call->filter); @@ -1938,6 +1943,13 @@ static int create_filter(struct ftrace_event_call *call, return err; } +int create_event_filter(struct ftrace_event_call *call, + char *filter_str, bool set_str, + struct event_filter **filterp) +{ + return create_filter(call, filter_str, set_str, filterp); +} + /** * create_system_filter - create a filter for an event_subsystem * @system: event_subsystem to create a filter for diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 45e48b109d51..f5b3f780fda4 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -31,6 +31,9 @@ static DEFINE_MUTEX(trigger_cmd_mutex); static void trigger_data_free(struct event_trigger_data *data) { + if (data->cmd_ops->set_filter) + data->cmd_ops->set_filter(NULL, data, NULL); + synchronize_sched(); /* make sure current triggers exit before free */ kfree(data); } @@ -38,27 +41,78 @@ trigger_data_free(struct event_trigger_data *data) /** * event_triggers_call - Call triggers associated with a trace event * @file: The ftrace_event_file associated with the event + * @rec: The trace entry for the event, NULL for unconditional invocation * * For each trigger associated with an event, invoke the trigger - * function registered with the associated trigger command. + * function registered with the associated trigger command. If rec is + * non-NULL, it means that the trigger requires further processing and + * shouldn't be unconditionally invoked. If rec is non-NULL and the + * trigger has a filter associated with it, rec will checked against + * the filter and if the record matches the trigger will be invoked. + * If the trigger is a 'post_trigger', meaning it shouldn't be invoked + * in any case until the current event is written, the trigger + * function isn't invoked but the bit associated with the deferred + * trigger is set in the return value. + * + * Returns an enum event_trigger_type value containing a set bit for + * any trigger that should be deferred, ETT_NONE if nothing to defer. * * Called from tracepoint handlers (with rcu_read_lock_sched() held). * * Return: an enum event_trigger_type value containing a set bit for * any trigger that should be deferred, ETT_NONE if nothing to defer. */ -void event_triggers_call(struct ftrace_event_file *file) +enum event_trigger_type +event_triggers_call(struct ftrace_event_file *file, void *rec) { struct event_trigger_data *data; + enum event_trigger_type tt = ETT_NONE; if (list_empty(&file->triggers)) - return; + return tt; - list_for_each_entry_rcu(data, &file->triggers, list) + list_for_each_entry_rcu(data, &file->triggers, list) { + if (!rec) { + data->ops->func(data); + continue; + } + if (data->filter && !filter_match_preds(data->filter, rec)) + continue; + if (data->cmd_ops->post_trigger) { + tt |= data->cmd_ops->trigger_type; + continue; + } data->ops->func(data); + } + return tt; } EXPORT_SYMBOL_GPL(event_triggers_call); +/** + * event_triggers_post_call - Call 'post_triggers' for a trace event + * @file: The ftrace_event_file associated with the event + * @tt: enum event_trigger_type containing a set bit for each trigger to invoke + * + * For each trigger associated with an event, invoke the trigger + * function registered with the associated trigger command, if the + * corresponding bit is set in the tt enum passed into this function. + * See @event_triggers_call for details on how those bits are set. + * + * Called from tracepoint handlers (with rcu_read_lock_sched() held). + */ +void +event_triggers_post_call(struct ftrace_event_file *file, + enum event_trigger_type tt) +{ + struct event_trigger_data *data; + + list_for_each_entry_rcu(data, &file->triggers, list) { + if (data->cmd_ops->trigger_type & tt) + data->ops->func(data); + } +} +EXPORT_SYMBOL_GPL(event_triggers_post_call); + static void *trigger_next(struct seq_file *m, void *t, loff_t *pos) { struct ftrace_event_file *event_file = event_file_data(m->private); @@ -403,6 +457,34 @@ clear_event_triggers(struct trace_array *tr) } /** + * update_cond_flag - Set or reset the TRIGGER_COND bit + * @file: The ftrace_event_file associated with the event + * + * If an event has triggers and any of those triggers has a filter or + * a post_trigger, trigger invocation needs to be deferred until after + * the current event has logged its data, and the event should have + * its TRIGGER_COND bit set, otherwise the TRIGGER_COND bit should be + * cleared. + */ +static void update_cond_flag(struct ftrace_event_file *file) +{ + struct event_trigger_data *data; + bool set_cond = false; + + list_for_each_entry_rcu(data, &file->triggers, list) { + if (data->filter || data->cmd_ops->post_trigger) { + set_cond = true; + break; + } + } + + if (set_cond) + set_bit(FTRACE_EVENT_FL_TRIGGER_COND_BIT, &file->flags); + else + clear_bit(FTRACE_EVENT_FL_TRIGGER_COND_BIT, &file->flags); +} + +/** * register_trigger - Generic event_command @reg implementation * @glob: The raw string used to register the trigger * @ops: The trigger ops associated with the trigger @@ -443,6 +525,7 @@ static int register_trigger(char *glob, struct event_trigger_ops *ops, list_del_rcu(&data->list); ret--; } + update_cond_flag(file); out: return ret; } @@ -470,6 +553,7 @@ static void unregister_trigger(char *glob, struct event_trigger_ops *ops, if (data->cmd_ops->trigger_type == test->cmd_ops->trigger_type) { unregistered = true; list_del_rcu(&data->list); + update_cond_flag(file); trace_event_trigger_enable_disable(file, 0); break; } @@ -572,10 +656,78 @@ event_trigger_callback(struct event_command *cmd_ops, return ret; out_free: + if (cmd_ops->set_filter) + cmd_ops->set_filter(NULL, trigger_data, NULL); kfree(trigger_data); goto out; } +/** + * set_trigger_filter - Generic event_command @set_filter implementation + * @filter_str: The filter string for the trigger, NULL to remove filter + * @trigger_data: Trigger-specific data + * @file: The ftrace_event_file associated with the event + * + * Common implementation for event command filter parsing and filter + * instantiation. + * + * Usually used directly as the @set_filter method in event command + * implementations. + * + * Also used to remove a filter (if filter_str = NULL). + * + * Return: 0 on success, errno otherwise + */ +static int set_trigger_filter(char *filter_str, + struct event_trigger_data *trigger_data, + struct ftrace_event_file *file) +{ + struct event_trigger_data *data = trigger_data; + struct event_filter *filter = NULL, *tmp; + int ret = -EINVAL; + char *s; + + if (!filter_str) /* clear the current filter */ + goto assign; + + s = strsep(&filter_str, " \t"); + + if (!strlen(s) || strcmp(s, "if") != 0) + goto out; + + if (!filter_str) + goto out; + + /* The filter is for the 'trigger' event, not the triggered event */ + ret = create_event_filter(file->event_call, filter_str, false, &filter); + if (ret) + goto out; + assign: + tmp = data->filter; + + rcu_assign_pointer(data->filter, filter); + + if (tmp) { + /* Make sure the call is done with the filter */ + synchronize_sched(); + free_event_filter(tmp); + } + + kfree(data->filter_str); + data->filter_str = NULL; + + if (filter_str) { + data->filter_str = kstrdup(filter_str, GFP_KERNEL); + if (!data->filter_str) { + free_event_filter(data->filter); + data->filter = NULL; + ret = -ENOMEM; + } + } + out: + return ret; +} + static void traceon_trigger(struct event_trigger_data *data) { @@ -685,6 +837,7 @@ static struct event_command trigger_traceon_cmd = { .reg = register_trigger, .unreg = unregister_trigger, .get_trigger_ops = onoff_get_trigger_ops, + .set_filter = set_trigger_filter, }; static struct event_command trigger_traceoff_cmd = { @@ -694,6 +847,7 @@ static struct event_command trigger_traceoff_cmd = { .reg = register_trigger, .unreg = unregister_trigger, .get_trigger_ops = onoff_get_trigger_ops, + .set_filter = set_trigger_filter, }; #ifdef CONFIG_TRACER_SNAPSHOT @@ -765,6 +919,7 @@ static struct event_command trigger_snapshot_cmd = { .reg = register_snapshot_trigger, .unreg = unregister_trigger, .get_trigger_ops = snapshot_get_trigger_ops, + .set_filter = set_trigger_filter, }; static __init int register_trigger_snapshot_cmd(void) @@ -843,6 +998,7 @@ static struct event_command trigger_stacktrace_cmd = { .reg = register_trigger, .unreg = unregister_trigger, .get_trigger_ops = stacktrace_get_trigger_ops, + .set_filter = set_trigger_filter, }; static __init int register_trigger_stacktrace_cmd(void) @@ -1100,6 +1256,8 @@ event_enable_trigger_func(struct event_command *cmd_ops, out_put: module_put(event_enable_file->event_call->mod); out_free: + if (cmd_ops->set_filter) + cmd_ops->set_filter(NULL, trigger_data, NULL); kfree(trigger_data); kfree(enable_data); goto out; @@ -1137,6 +1295,7 @@ static int event_enable_register_trigger(char *glob, list_del_rcu(&data->list); ret--; } + update_cond_flag(file); out: return ret; } @@ -1157,6 +1316,7 @@ static void event_enable_unregister_trigger(char *glob, (enable_data->file == test_enable_data->file)) { unregistered = true; list_del_rcu(&data->list); + update_cond_flag(file); trace_event_trigger_enable_disable(file, 0); break; } @@ -1191,6 +1351,7 @@ static struct event_command trigger_enable_cmd = { .reg = event_enable_register_trigger, .unreg = event_enable_unregister_trigger, .get_trigger_ops = event_enable_get_trigger_ops, + .set_filter = set_trigger_filter, }; static struct event_command trigger_disable_cmd = { @@ -1200,6 +1361,7 @@ static struct event_command trigger_disable_cmd = { .reg = event_enable_register_trigger, .unreg = event_enable_unregister_trigger, .get_trigger_ops = event_enable_get_trigger_ops, + .set_filter = set_trigger_filter, }; static __init void unregister_trigger_enable_disable_cmds(void) diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 936ec3960335..fdd955f2f1aa 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -306,8 +306,10 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id) struct syscall_trace_enter *entry; struct syscall_metadata *sys_data; struct ring_buffer_event *event; + enum event_trigger_type __tt = ETT_NONE; struct ring_buffer *buffer; unsigned long irq_flags; + unsigned long eflags; int pc; int syscall_nr; int size; @@ -321,10 +323,14 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id) if (!ftrace_file) return; - if (test_bit(FTRACE_EVENT_FL_TRIGGER_MODE_BIT, &ftrace_file->flags)) - event_triggers_call(ftrace_file); - if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags)) - return; + eflags = ftrace_file->flags; + + if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) { + if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE) + event_triggers_call(ftrace_file, NULL); + if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED) + return; + } sys_data = syscall_nr_to_meta(syscall_nr); if (!sys_data) @@ -345,9 +351,16 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id) entry->nr = syscall_nr; syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args); - if (!filter_check_discard(ftrace_file, entry, buffer, event)) + if (eflags & FTRACE_EVENT_FL_TRIGGER_COND) + __tt = event_triggers_call(ftrace_file, entry); + + if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags)) + ring_buffer_discard_commit(buffer, event); + else if (!filter_check_discard(ftrace_file, entry, buffer, event)) trace_current_buffer_unlock_commit(buffer, event, irq_flags, pc); + if (__tt) + event_triggers_post_call(ftrace_file, __tt); } static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) @@ -357,8 +370,10 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) struct syscall_trace_exit *entry; struct syscall_metadata *sys_data; struct ring_buffer_event *event; + enum event_trigger_type __tt = ETT_NONE; struct ring_buffer *buffer; unsigned long irq_flags; + unsigned long eflags; int pc; int syscall_nr; @@ -371,10 +386,14 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) if (!ftrace_file) return; - if (test_bit(FTRACE_EVENT_FL_TRIGGER_MODE_BIT, &ftrace_file->flags)) - event_triggers_call(ftrace_file); - if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags)) - return; + eflags = ftrace_file->flags; + + if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) { + if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE) + event_triggers_call(ftrace_file, NULL); + if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED) + return; + } sys_data = syscall_nr_to_meta(syscall_nr); if (!sys_data) @@ -394,9 +413,16 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) entry->nr = syscall_nr; entry->ret = syscall_get_return_value(current, regs); - if (!filter_check_discard(ftrace_file, entry, buffer, event)) + if (eflags & FTRACE_EVENT_FL_TRIGGER_COND) + __tt = event_triggers_call(ftrace_file, entry); + + if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &ftrace_file->flags)) + ring_buffer_discard_commit(buffer, event); + else if (!filter_check_discard(ftrace_file, entry, buffer, event)) trace_current_buffer_unlock_commit(buffer, event, irq_flags, pc); + if (__tt) + event_triggers_post_call(ftrace_file, __tt); } static int reg_event_syscall_enter(struct ftrace_event_file *file, |