linux/kernel/trace/trace_events_hist.c

// SPDX-License-Identifier: GPL-2.0
/*
 * trace_events_hist - trace event hist triggers
 *
 * Copyright (C) 2015 Tom Zanussi <[email protected]>
 */

#include <linux/module.h>
#include <linux/kallsyms.h>
#include <linux/security.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/rculist.h>
#include <linux/tracefs.h>

/* for gfp flag names */
#include <linux/trace_events.h>
#include <trace/events/mmflags.h>

#include "tracing_map.h"
#include "trace_synth.h"

#define ERRORS …

#undef C
#define C(a, b) …

enum { … };

#undef C
#define C(a, b) …

static const char *err_text[] = …;

struct hist_field;

hist_field_fn_t;

#define HIST_FIELD_OPERANDS_MAX …
#define HIST_FIELDS_MAX …
#define HIST_ACTIONS_MAX …
#define HIST_CONST_DIGITS_MAX …
#define HIST_DIV_SHIFT …

enum field_op_id { … };

enum hist_field_fn { … };

/*
 * A hist_var (histogram variable) contains variable information for
 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 * flag set.  A hist_var has a variable name e.g. ts0, and is
 * associated with a given histogram trigger, as specified by
 * hist_data.  The hist_var idx is the unique index assigned to the
 * variable by the hist trigger's tracing_map.  The idx is what is
 * used to set a variable's value and, by a variable reference, to
 * retrieve it.
 */
struct hist_var { … };

struct hist_field { … };

static u64 hist_fn_call(struct hist_field *hist_field,
			struct tracing_map_elt *elt,
			struct trace_buffer *buffer,
			struct ring_buffer_event *rbe,
			void *event);

static u64 hist_field_const(struct hist_field *field,
			   struct tracing_map_elt *elt,
			   struct trace_buffer *buffer,
			   struct ring_buffer_event *rbe,
			   void *event)
{ … }

static u64 hist_field_counter(struct hist_field *field,
			      struct tracing_map_elt *elt,
			      struct trace_buffer *buffer,
			      struct ring_buffer_event *rbe,
			      void *event)
{ … }

static u64 hist_field_string(struct hist_field *hist_field,
			     struct tracing_map_elt *elt,
			     struct trace_buffer *buffer,
			     struct ring_buffer_event *rbe,
			     void *event)
{ … }

static u64 hist_field_dynstring(struct hist_field *hist_field,
				struct tracing_map_elt *elt,
				struct trace_buffer *buffer,
				struct ring_buffer_event *rbe,
				void *event)
{ … }

static u64 hist_field_reldynstring(struct hist_field *hist_field,
				   struct tracing_map_elt *elt,
				   struct trace_buffer *buffer,
				   struct ring_buffer_event *rbe,
				   void *event)
{ … }

static u64 hist_field_pstring(struct hist_field *hist_field,
			      struct tracing_map_elt *elt,
			      struct trace_buffer *buffer,
			      struct ring_buffer_event *rbe,
			      void *event)
{ … }

static u64 hist_field_log2(struct hist_field *hist_field,
			   struct tracing_map_elt *elt,
			   struct trace_buffer *buffer,
			   struct ring_buffer_event *rbe,
			   void *event)
{ … }

static u64 hist_field_bucket(struct hist_field *hist_field,
			     struct tracing_map_elt *elt,
			     struct trace_buffer *buffer,
			     struct ring_buffer_event *rbe,
			     void *event)
{ … }

static u64 hist_field_plus(struct hist_field *hist_field,
			   struct tracing_map_elt *elt,
			   struct trace_buffer *buffer,
			   struct ring_buffer_event *rbe,
			   void *event)
{ … }

static u64 hist_field_minus(struct hist_field *hist_field,
			    struct tracing_map_elt *elt,
			    struct trace_buffer *buffer,
			    struct ring_buffer_event *rbe,
			    void *event)
{ … }

static u64 hist_field_div(struct hist_field *hist_field,
			   struct tracing_map_elt *elt,
			   struct trace_buffer *buffer,
			   struct ring_buffer_event *rbe,
			   void *event)
{ … }

static u64 div_by_power_of_two(struct hist_field *hist_field,
				struct tracing_map_elt *elt,
				struct trace_buffer *buffer,
				struct ring_buffer_event *rbe,
				void *event)
{ … }

static u64 div_by_not_power_of_two(struct hist_field *hist_field,
				struct tracing_map_elt *elt,
				struct trace_buffer *buffer,
				struct ring_buffer_event *rbe,
				void *event)
{ … }

static u64 div_by_mult_and_shift(struct hist_field *hist_field,
				struct tracing_map_elt *elt,
				struct trace_buffer *buffer,
				struct ring_buffer_event *rbe,
				void *event)
{ … }

static u64 hist_field_mult(struct hist_field *hist_field,
			   struct tracing_map_elt *elt,
			   struct trace_buffer *buffer,
			   struct ring_buffer_event *rbe,
			   void *event)
{ … }

static u64 hist_field_unary_minus(struct hist_field *hist_field,
				  struct tracing_map_elt *elt,
				  struct trace_buffer *buffer,
				  struct ring_buffer_event *rbe,
				  void *event)
{ … }

#define DEFINE_HIST_FIELD_FN(type) …

DEFINE_HIST_FIELD_FN(s64);
DEFINE_HIST_FIELD_FN(u64);
DEFINE_HIST_FIELD_FN(s32);
DEFINE_HIST_FIELD_FN(u32);
DEFINE_HIST_FIELD_FN(s16);
DEFINE_HIST_FIELD_FN(u16);
DEFINE_HIST_FIELD_FN(s8);
DEFINE_HIST_FIELD_FN(u8);

#define for_each_hist_field(i, hist_data) …

#define for_each_hist_val_field(i, hist_data) …

#define for_each_hist_key_field(i, hist_data) …

#define HITCOUNT_IDX …
#define HIST_KEY_SIZE_MAX …

enum hist_field_flags { … };

struct var_defs { … };

struct hist_trigger_attrs { … };

struct field_var { … };

struct field_var_hist { … };

struct hist_trigger_data { … };

struct action_data;

action_fn_t;

check_track_val_fn_t;

enum handler_id { … };

enum action_id { … };

struct action_data { … };

struct track_data { … };

struct hist_elt_data { … };

struct snapshot_context { … };

/*
 * Returns the specific division function to use if the divisor
 * is constant. This avoids extra branches when the trigger is hit.
 */
static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
{ … }

static void track_data_free(struct track_data *track_data)
{ … }

static struct track_data *track_data_alloc(unsigned int key_len,
					   struct action_data *action_data,
					   struct hist_trigger_data *hist_data)
{ … }

#define HIST_PREFIX …

static char *last_cmd;
static char last_cmd_loc[MAX_FILTER_STR_VAL];

static int errpos(char *str)
{ … }

static void last_cmd_set(struct trace_event_file *file, char *str)
{ … }

static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
{ … }

static void hist_err_clear(void)
{ … }

synth_probe_func_t;

static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
			       unsigned int *var_ref_idx)
{ … }

static void action_trace(struct hist_trigger_data *hist_data,
			 struct tracing_map_elt *elt,
			 struct trace_buffer *buffer, void *rec,
			 struct ring_buffer_event *rbe, void *key,
			 struct action_data *data, u64 *var_ref_vals)
{ … }

struct hist_var_data { … };

static u64 hist_field_timestamp(struct hist_field *hist_field,
				struct tracing_map_elt *elt,
				struct trace_buffer *buffer,
				struct ring_buffer_event *rbe,
				void *event)
{ … }

static u64 hist_field_cpu(struct hist_field *hist_field,
			  struct tracing_map_elt *elt,
			  struct trace_buffer *buffer,
			  struct ring_buffer_event *rbe,
			  void *event)
{ … }

/**
 * check_field_for_var_ref - Check if a VAR_REF field references a variable
 * @hist_field: The VAR_REF field to check
 * @var_data: The hist trigger that owns the variable
 * @var_idx: The trigger variable identifier
 *
 * Check the given VAR_REF field to see whether or not it references
 * the given variable associated with the given trigger.
 *
 * Return: The VAR_REF field if it does reference the variable, NULL if not
 */
static struct hist_field *
check_field_for_var_ref(struct hist_field *hist_field,
			struct hist_trigger_data *var_data,
			unsigned int var_idx)
{ … }

/**
 * find_var_ref - Check if a trigger has a reference to a trigger variable
 * @hist_data: The hist trigger that might have a reference to the variable
 * @var_data: The hist trigger that owns the variable
 * @var_idx: The trigger variable identifier
 *
 * Check the list of var_refs[] on the first hist trigger to see
 * whether any of them are references to the variable on the second
 * trigger.
 *
 * Return: The VAR_REF field referencing the variable if so, NULL if not
 */
static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
				       struct hist_trigger_data *var_data,
				       unsigned int var_idx)
{ … }

/**
 * find_any_var_ref - Check if there is a reference to a given trigger variable
 * @hist_data: The hist trigger
 * @var_idx: The trigger variable identifier
 *
 * Check to see whether the given variable is currently referenced by
 * any other trigger.
 *
 * The trigger the variable is defined on is explicitly excluded - the
 * assumption being that a self-reference doesn't prevent a trigger
 * from being removed.
 *
 * Return: The VAR_REF field referencing the variable if so, NULL if not
 */
static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
					   unsigned int var_idx)
{ … }

/**
 * check_var_refs - Check if there is a reference to any of trigger's variables
 * @hist_data: The hist trigger
 *
 * A trigger can define one or more variables.  If any one of them is
 * currently referenced by any other trigger, this function will
 * determine that.
 *
 * Typically used to determine whether or not a trigger can be removed
 * - if there are any references to a trigger's variables, it cannot.
 *
 * Return: True if there is a reference to any of trigger's variables
 */
static bool check_var_refs(struct hist_trigger_data *hist_data)
{ … }

static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
{ … }

static bool field_has_hist_vars(struct hist_field *hist_field,
				unsigned int level)
{ … }

static bool has_hist_vars(struct hist_trigger_data *hist_data)
{ … }

static int save_hist_vars(struct hist_trigger_data *hist_data)
{ … }

static void remove_hist_vars(struct hist_trigger_data *hist_data)
{ … }

static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
					 const char *var_name)
{ … }

static struct hist_field *find_var(struct hist_trigger_data *hist_data,
				   struct trace_event_file *file,
				   const char *var_name)
{ … }

static struct trace_event_file *find_var_file(struct trace_array *tr,
					      char *system,
					      char *event_name,
					      char *var_name)
{ … }

static struct hist_field *find_file_var(struct trace_event_file *file,
					const char *var_name)
{ … }

static struct hist_field *
find_match_var(struct hist_trigger_data *hist_data, char *var_name)
{ … }

static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
					 char *system,
					 char *event_name,
					 char *var_name)
{ … }

static u64 hist_field_var_ref(struct hist_field *hist_field,
			      struct tracing_map_elt *elt,
			      struct trace_buffer *buffer,
			      struct ring_buffer_event *rbe,
			      void *event)
{ … }

static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
			     u64 *var_ref_vals, bool self)
{ … }

static const char *hist_field_name(struct hist_field *field,
				   unsigned int level)
{ … }

static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
{ … }

static int parse_map_size(char *str)
{ … }

static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
{ … }

static int parse_action(char *str, struct hist_trigger_attrs *attrs)
{ … }

static int parse_assignment(struct trace_array *tr,
			    char *str, struct hist_trigger_attrs *attrs)
{ … }

static struct hist_trigger_attrs *
parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
{ … }

static inline void save_comm(char *comm, struct task_struct *task)
{ … }

static void hist_elt_data_free(struct hist_elt_data *elt_data)
{ … }

static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
{ … }

static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
{ … }

static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
{ … }

static const struct tracing_map_ops hist_trigger_elt_data_ops = …;

static const char *get_hist_field_flags(struct hist_field *hist_field)
{ … }

static void expr_field_str(struct hist_field *field, char *expr)
{ … }

static char *expr_str(struct hist_field *field, unsigned int level)
{ … }

/*
 * If field_op != FIELD_OP_NONE, *sep points to the root operator
 * of the expression tree to be evaluated.
 */
static int contains_operator(char *str, char **sep)
{ … }

static void get_hist_field(struct hist_field *hist_field)
{ … }

static void __destroy_hist_field(struct hist_field *hist_field)
{ … }

static void destroy_hist_field(struct hist_field *hist_field,
			       unsigned int level)
{ … }

static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
					    struct ftrace_event_field *field,
					    unsigned long flags,
					    char *var_name)
{ … }

static void destroy_hist_fields(struct hist_trigger_data *hist_data)
{ … }

static int init_var_ref(struct hist_field *ref_field,
			struct hist_field *var_field,
			char *system, char *event_name)
{ … }

static int find_var_ref_idx(struct hist_trigger_data *hist_data,
			    struct hist_field *var_field)
{ … }

/**
 * create_var_ref - Create a variable reference and attach it to trigger
 * @hist_data: The trigger that will be referencing the variable
 * @var_field: The VAR field to create a reference to
 * @system: The optional system string
 * @event_name: The optional event_name string
 *
 * Given a variable hist_field, create a VAR_REF hist_field that
 * represents a reference to it.
 *
 * This function also adds the reference to the trigger that
 * now references the variable.
 *
 * Return: The VAR_REF field if successful, NULL if not
 */
static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
					 struct hist_field *var_field,
					 char *system, char *event_name)
{ … }

static bool is_var_ref(char *var_name)
{ … }

static char *field_name_from_var(struct hist_trigger_data *hist_data,
				 char *var_name)
{ … }

static char *local_field_var_ref(struct hist_trigger_data *hist_data,
				 char *system, char *event_name,
				 char *var_name)
{ … }

static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
					char *system, char *event_name,
					char *var_name)
{ … }

static struct ftrace_event_field *
parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
	    char *field_str, unsigned long *flags, unsigned long *buckets)
{ … }

static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
				       struct hist_field *var_ref,
				       char *var_name)
{ … }

static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
				      char *str, char *var_name,
				      unsigned long *flags)
{ … }

static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
				     struct trace_event_file *file, char *str,
				     unsigned long *flags, char *var_name)
{ … }

static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
				     struct trace_event_file *file,
				     char *str, unsigned long flags,
				     char *var_name, unsigned int *n_subexprs);

static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
				      struct trace_event_file *file,
				      char *str, unsigned long flags,
				      char *var_name, unsigned int *n_subexprs)
{ … }

/*
 * If the operands are var refs, return pointers the
 * variable(s) referenced in var1 and var2, else NULL.
 */
static int check_expr_operands(struct trace_array *tr,
			       struct hist_field *operand1,
			       struct hist_field *operand2,
			       struct hist_field **var1,
			       struct hist_field **var2)
{ … }

static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
				     struct trace_event_file *file,
				     char *str, unsigned long flags,
				     char *var_name, unsigned int *n_subexprs)
{ … }

static char *find_trigger_filter(struct hist_trigger_data *hist_data,
				 struct trace_event_file *file)
{ … }

static struct event_command trigger_hist_cmd;
static int event_hist_trigger_parse(struct event_command *cmd_ops,
				    struct trace_event_file *file,
				    char *glob, char *cmd,
				    char *param_and_filter);

static bool compatible_keys(struct hist_trigger_data *target_hist_data,
			    struct hist_trigger_data *hist_data,
			    unsigned int n_keys)
{ … }

static struct hist_trigger_data *
find_compatible_hist(struct hist_trigger_data *target_hist_data,
		     struct trace_event_file *file)
{ … }

static struct trace_event_file *event_file(struct trace_array *tr,
					   char *system, char *event_name)
{ … }

static struct hist_field *
find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
			 char *system, char *event_name, char *field_name)
{ … }

/**
 * create_field_var_hist - Automatically create a histogram and var for a field
 * @target_hist_data: The target hist trigger
 * @subsys_name: Optional subsystem name
 * @event_name: Optional event name
 * @field_name: The name of the field (and the resulting variable)
 *
 * Hist trigger actions fetch data from variables, not directly from
 * events.  However, for convenience, users are allowed to directly
 * specify an event field in an action, which will be automatically
 * converted into a variable on their behalf.
 *
 * If a user specifies a field on an event that isn't the event the
 * histogram currently being defined (the target event histogram), the
 * only way that can be accomplished is if a new hist trigger is
 * created and the field variable defined on that.
 *
 * This function creates a new histogram compatible with the target
 * event (meaning a histogram with the same key as the target
 * histogram), and creates a variable for the specified field, but
 * with 'synthetic_' prepended to the variable name in order to avoid
 * collision with normal field variables.
 *
 * Return: The variable created for the field.
 */
static struct hist_field *
create_field_var_hist(struct hist_trigger_data *target_hist_data,
		      char *subsys_name, char *event_name, char *field_name)
{ … }

static struct hist_field *
find_target_event_var(struct hist_trigger_data *hist_data,
		      char *subsys_name, char *event_name, char *var_name)
{ … }

static inline void __update_field_vars(struct tracing_map_elt *elt,
				       struct trace_buffer *buffer,
				       struct ring_buffer_event *rbe,
				       void *rec,
				       struct field_var **field_vars,
				       unsigned int n_field_vars,
				       unsigned int field_var_str_start)
{ … }

static void update_field_vars(struct hist_trigger_data *hist_data,
			      struct tracing_map_elt *elt,
			      struct trace_buffer *buffer,
			      struct ring_buffer_event *rbe,
			      void *rec)
{ … }

static void save_track_data_vars(struct hist_trigger_data *hist_data,
				 struct tracing_map_elt *elt,
				 struct trace_buffer *buffer,  void *rec,
				 struct ring_buffer_event *rbe, void *key,
				 struct action_data *data, u64 *var_ref_vals)
{ … }

static struct hist_field *create_var(struct hist_trigger_data *hist_data,
				     struct trace_event_file *file,
				     char *name, int size, const char *type)
{ … }

static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
					  struct trace_event_file *file,
					  char *field_name)
{ … }

/**
 * create_target_field_var - Automatically create a variable for a field
 * @target_hist_data: The target hist trigger
 * @subsys_name: Optional subsystem name
 * @event_name: Optional event name
 * @var_name: The name of the field (and the resulting variable)
 *
 * Hist trigger actions fetch data from variables, not directly from
 * events.  However, for convenience, users are allowed to directly
 * specify an event field in an action, which will be automatically
 * converted into a variable on their behalf.
 *
 * This function creates a field variable with the name var_name on
 * the hist trigger currently being defined on the target event.  If
 * subsys_name and event_name are specified, this function simply
 * verifies that they do in fact match the target event subsystem and
 * event name.
 *
 * Return: The variable created for the field.
 */
static struct field_var *
create_target_field_var(struct hist_trigger_data *target_hist_data,
			char *subsys_name, char *event_name, char *var_name)
{ … }

static bool check_track_val_max(u64 track_val, u64 var_val)
{ … }

static bool check_track_val_changed(u64 track_val, u64 var_val)
{ … }

static u64 get_track_val(struct hist_trigger_data *hist_data,
			 struct tracing_map_elt *elt,
			 struct action_data *data)
{ … }

static void save_track_val(struct hist_trigger_data *hist_data,
			   struct tracing_map_elt *elt,
			   struct action_data *data, u64 var_val)
{ … }

static void save_track_data(struct hist_trigger_data *hist_data,
			    struct tracing_map_elt *elt,
			    struct trace_buffer *buffer, void *rec,
			    struct ring_buffer_event *rbe, void *key,
			    struct action_data *data, u64 *var_ref_vals)
{ … }

static bool check_track_val(struct tracing_map_elt *elt,
			    struct action_data *data,
			    u64 var_val)
{ … }

#ifdef CONFIG_TRACER_SNAPSHOT
static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
{ … }

static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
				     struct tracing_map_elt *elt,
				     struct trace_buffer *buffer, void *rec,
				     struct ring_buffer_event *rbe, void *key,
				     struct action_data *data,
				     u64 *var_ref_vals)
{ … }

static void hist_trigger_print_key(struct seq_file *m,
				   struct hist_trigger_data *hist_data,
				   void *key,
				   struct tracing_map_elt *elt);

static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
{ … }

static void track_data_snapshot_print(struct seq_file *m,
				      struct hist_trigger_data *hist_data)
{ … }
#else
static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
{
	return false;
}
static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
				     struct tracing_map_elt *elt,
				     struct trace_buffer *buffer, void *rec,
				     struct ring_buffer_event *rbe, void *key,
				     struct action_data *data,
				     u64 *var_ref_vals) {}
static void track_data_snapshot_print(struct seq_file *m,
				      struct hist_trigger_data *hist_data) {}
#endif /* CONFIG_TRACER_SNAPSHOT */

static void track_data_print(struct seq_file *m,
			     struct hist_trigger_data *hist_data,
			     struct tracing_map_elt *elt,
			     struct action_data *data)
{ … }

static void ontrack_action(struct hist_trigger_data *hist_data,
			   struct tracing_map_elt *elt,
			   struct trace_buffer *buffer, void *rec,
			   struct ring_buffer_event *rbe, void *key,
			   struct action_data *data, u64 *var_ref_vals)
{ … }

static void action_data_destroy(struct action_data *data)
{ … }

static void track_data_destroy(struct hist_trigger_data *hist_data,
			       struct action_data *data)
{ … }

static int action_create(struct hist_trigger_data *hist_data,
			 struct action_data *data);

static int track_data_create(struct hist_trigger_data *hist_data,
			     struct action_data *data)
{ … }

static int parse_action_params(struct trace_array *tr, char *params,
			       struct action_data *data)
{ … }

static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
			enum handler_id handler)
{ … }

static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
					    char *str, enum handler_id handler)
{ … }

static void onmatch_destroy(struct action_data *data)
{ … }

static void destroy_field_var(struct field_var *field_var)
{ … }

static void destroy_field_vars(struct hist_trigger_data *hist_data)
{ … }

static void save_field_var(struct hist_trigger_data *hist_data,
			   struct field_var *field_var)
{ … }


static int check_synth_field(struct synth_event *event,
			     struct hist_field *hist_field,
			     unsigned int field_pos)
{ … }

static struct hist_field *
trace_action_find_var(struct hist_trigger_data *hist_data,
		      struct action_data *data,
		      char *system, char *event, char *var)
{ … }

static struct hist_field *
trace_action_create_field_var(struct hist_trigger_data *hist_data,
			      struct action_data *data, char *system,
			      char *event, char *var)
{ … }

static int trace_action_create(struct hist_trigger_data *hist_data,
			       struct action_data *data)
{ … }

static int action_create(struct hist_trigger_data *hist_data,
			 struct action_data *data)
{ … }

static int onmatch_create(struct hist_trigger_data *hist_data,
			  struct action_data *data)
{ … }

static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
{ … }

static int create_hitcount_val(struct hist_trigger_data *hist_data)
{ … }

static int __create_val_field(struct hist_trigger_data *hist_data,
			      unsigned int val_idx,
			      struct trace_event_file *file,
			      char *var_name, char *field_str,
			      unsigned long flags)
{ … }

static int create_val_field(struct hist_trigger_data *hist_data,
			    unsigned int val_idx,
			    struct trace_event_file *file,
			    char *field_str)
{ … }

static const char no_comm[] = …;

static u64 hist_field_execname(struct hist_field *hist_field,
			       struct tracing_map_elt *elt,
			       struct trace_buffer *buffer,
			       struct ring_buffer_event *rbe,
			       void *event)
{ … }

static u64 hist_field_stack(struct hist_field *hist_field,
			    struct tracing_map_elt *elt,
			    struct trace_buffer *buffer,
			    struct ring_buffer_event *rbe,
			    void *event)
{ … }

static u64 hist_fn_call(struct hist_field *hist_field,
			struct tracing_map_elt *elt,
			struct trace_buffer *buffer,
			struct ring_buffer_event *rbe,
			void *event)
{ … }

/* Convert a var that points to common_pid.execname to a string */
static void update_var_execname(struct hist_field *hist_field)
{ … }

static int create_var_field(struct hist_trigger_data *hist_data,
			    unsigned int val_idx,
			    struct trace_event_file *file,
			    char *var_name, char *expr_str)
{ … }

static int create_val_fields(struct hist_trigger_data *hist_data,
			     struct trace_event_file *file)
{ … }

static int create_key_field(struct hist_trigger_data *hist_data,
			    unsigned int key_idx,
			    unsigned int key_offset,
			    struct trace_event_file *file,
			    char *field_str)
{ … }

static int create_key_fields(struct hist_trigger_data *hist_data,
			     struct trace_event_file *file)
{ … }

static int create_var_fields(struct hist_trigger_data *hist_data,
			     struct trace_event_file *file)
{ … }

static void free_var_defs(struct hist_trigger_data *hist_data)
{ … }

static int parse_var_defs(struct hist_trigger_data *hist_data)
{ … }

static int create_hist_fields(struct hist_trigger_data *hist_data,
			      struct trace_event_file *file)
{ … }

static int is_descending(struct trace_array *tr, const char *str)
{ … }

static int create_sort_keys(struct hist_trigger_data *hist_data)
{ … }

static void destroy_actions(struct hist_trigger_data *hist_data)
{ … }

static int parse_actions(struct hist_trigger_data *hist_data)
{ … }

static int create_actions(struct hist_trigger_data *hist_data)
{ … }

static void print_actions(struct seq_file *m,
			  struct hist_trigger_data *hist_data,
			  struct tracing_map_elt *elt)
{ … }

static void print_action_spec(struct seq_file *m,
			      struct hist_trigger_data *hist_data,
			      struct action_data *data)
{ … }

static void print_track_data_spec(struct seq_file *m,
				  struct hist_trigger_data *hist_data,
				  struct action_data *data)
{ … }

static void print_onmatch_spec(struct seq_file *m,
			       struct hist_trigger_data *hist_data,
			       struct action_data *data)
{ … }

static bool actions_match(struct hist_trigger_data *hist_data,
			  struct hist_trigger_data *hist_data_test)
{ … }


static void print_actions_spec(struct seq_file *m,
			       struct hist_trigger_data *hist_data)
{ … }

static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
{ … }

static void destroy_hist_data(struct hist_trigger_data *hist_data)
{ … }

static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
{ … }

static struct hist_trigger_data *
create_hist_data(unsigned int map_bits,
		 struct hist_trigger_attrs *attrs,
		 struct trace_event_file *file,
		 bool remove)
{ … }

static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
				    struct tracing_map_elt *elt,
				    struct trace_buffer *buffer, void *rec,
				    struct ring_buffer_event *rbe,
				    u64 *var_ref_vals)
{ … }

static inline void add_to_key(char *compound_key, void *key,
			      struct hist_field *key_field, void *rec)
{ … }

static void
hist_trigger_actions(struct hist_trigger_data *hist_data,
		     struct tracing_map_elt *elt,
		     struct trace_buffer *buffer, void *rec,
		     struct ring_buffer_event *rbe, void *key,
		     u64 *var_ref_vals)
{ … }

static void event_hist_trigger(struct event_trigger_data *data,
			       struct trace_buffer *buffer, void *rec,
			       struct ring_buffer_event *rbe)
{ … }

static void hist_trigger_stacktrace_print(struct seq_file *m,
					  unsigned long *stacktrace_entries,
					  unsigned int max_entries)
{ … }

static void hist_trigger_print_key(struct seq_file *m,
				   struct hist_trigger_data *hist_data,
				   void *key,
				   struct tracing_map_elt *elt)
{ … }

/* Get the 100 times of the percentage of @val in @total */
static inline unsigned int __get_percentage(u64 val, u64 total)
{ … }

#define BAR_CHAR …

static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
{ … }

struct hist_val_stat { … };

static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
				   const char *field_name, unsigned long flags,
				   struct hist_val_stat *stats,
				   struct tracing_map_elt *elt)
{ … }

static void hist_trigger_entry_print(struct seq_file *m,
				     struct hist_trigger_data *hist_data,
				     struct hist_val_stat *stats,
				     void *key,
				     struct tracing_map_elt *elt)
{ … }

static int print_entries(struct seq_file *m,
			 struct hist_trigger_data *hist_data)
{ … }

static void hist_trigger_show(struct seq_file *m,
			      struct event_trigger_data *data, int n)
{ … }

static int hist_show(struct seq_file *m, void *v)
{ … }

static int event_hist_open(struct inode *inode, struct file *file)
{ … }

const struct file_operations event_hist_fops = …;

#ifdef CONFIG_HIST_TRIGGERS_DEBUG
static void hist_field_debug_show_flags(struct seq_file *m,
					unsigned long flags)
{ … }

static int hist_field_debug_show(struct seq_file *m,
				 struct hist_field *field, unsigned long flags)
{ … }

static int field_var_debug_show(struct seq_file *m,
				struct field_var *field_var, unsigned int i,
				bool save_vars)
{ … }

static int hist_action_debug_show(struct seq_file *m,
				  struct action_data *data, int i)
{ … }

static int hist_actions_debug_show(struct seq_file *m,
				   struct hist_trigger_data *hist_data)
{ … }

static void hist_trigger_debug_show(struct seq_file *m,
				    struct event_trigger_data *data, int n)
{ … }

static int hist_debug_show(struct seq_file *m, void *v)
{ … }

static int event_hist_debug_open(struct inode *inode, struct file *file)
{ … }

const struct file_operations event_hist_debug_fops = …;
#endif

static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
{ … }

static int event_hist_trigger_print(struct seq_file *m,
				    struct event_trigger_data *data)
{ … }

static int event_hist_trigger_init(struct event_trigger_data *data)
{ … }

static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
{ … }

static void event_hist_trigger_free(struct event_trigger_data *data)
{ … }

static struct event_trigger_ops event_hist_trigger_ops = …;

static int event_hist_trigger_named_init(struct event_trigger_data *data)
{ … }

static void event_hist_trigger_named_free(struct event_trigger_data *data)
{ … }

static struct event_trigger_ops event_hist_trigger_named_ops = …;

static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
							    char *param)
{ … }

static void hist_clear(struct event_trigger_data *data)
{ … }

static bool compatible_field(struct ftrace_event_field *field,
			     struct ftrace_event_field *test_field)
{ … }

static bool hist_trigger_match(struct event_trigger_data *data,
			       struct event_trigger_data *data_test,
			       struct event_trigger_data *named_data,
			       bool ignore_filter)
{ … }

static bool existing_hist_update_only(char *glob,
				      struct event_trigger_data *data,
				      struct trace_event_file *file)
{ … }

static int hist_register_trigger(char *glob,
				 struct event_trigger_data *data,
				 struct trace_event_file *file)
{ … }

static int hist_trigger_enable(struct event_trigger_data *data,
			       struct trace_event_file *file)
{ … }

static bool have_hist_trigger_match(struct event_trigger_data *data,
				    struct trace_event_file *file)
{ … }

static bool hist_trigger_check_refs(struct event_trigger_data *data,
				    struct trace_event_file *file)
{ … }

static void hist_unregister_trigger(char *glob,
				    struct event_trigger_data *data,
				    struct trace_event_file *file)
{ … }

static bool hist_file_check_refs(struct trace_event_file *file)
{ … }

static void hist_unreg_all(struct trace_event_file *file)
{ … }

static int event_hist_trigger_parse(struct event_command *cmd_ops,
				    struct trace_event_file *file,
				    char *glob, char *cmd,
				    char *param_and_filter)
{ … }

static struct event_command trigger_hist_cmd = …;

__init int register_trigger_hist_cmd(void)
{ … }

static void
hist_enable_trigger(struct event_trigger_data *data,
		    struct trace_buffer *buffer,  void *rec,
		    struct ring_buffer_event *event)
{ … }

static void
hist_enable_count_trigger(struct event_trigger_data *data,
			  struct trace_buffer *buffer,  void *rec,
			  struct ring_buffer_event *event)
{ … }

static struct event_trigger_ops hist_enable_trigger_ops = …;

static struct event_trigger_ops hist_enable_count_trigger_ops = …;

static struct event_trigger_ops hist_disable_trigger_ops = …;

static struct event_trigger_ops hist_disable_count_trigger_ops = …;

static struct event_trigger_ops *
hist_enable_get_trigger_ops(char *cmd, char *param)
{ … }

static void hist_enable_unreg_all(struct trace_event_file *file)
{ … }

static struct event_command trigger_hist_enable_cmd = …;

static struct event_command trigger_hist_disable_cmd = …;

static __init void unregister_trigger_hist_enable_disable_cmds(void)
{ … }

__init int register_trigger_hist_enable_disable_cmds(void)
{ … }