Codebase Browser
linux
Go to App

linux/kernel/trace/trace_probe.c 

// SPDX-License-Identifier: GPL-2.0
/*
 * Common code for probe-based Dynamic events.
 *
 * This code was copied from kernel/trace/trace_kprobe.c written by
 * Masami Hiramatsu <[email protected]>
 *
 * Updates to make this generic:
 * Copyright (C) IBM Corporation, 2010-2011
 * Author:     Srikar Dronamraju
 */
#define pr_fmt(fmt) …

#include <linux/bpf.h>
#include <linux/fs.h>
#include "trace_btf.h"

#include "trace_probe.h"

#undef C
#define C(a, b) …

static const char *trace_probe_err_text[] = …;

static const char *reserved_field_names[] = …;

/* Printing  in basic type function template */
#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) …

DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")

int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
{ … }
const char PRINT_TYPE_FMT_NAME(symbol)[] = …;

/* Print type function for string type */
int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
{ … }

const char PRINT_TYPE_FMT_NAME(string)[] = …;

/* Fetch type information table */
static const struct fetch_type probe_fetch_types[] = …;

static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
{ … }

static struct trace_probe_log trace_probe_log;

void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
{ … }

void trace_probe_log_clear(void)
{ … }

void trace_probe_log_set_index(int index)
{ … }

void __trace_probe_log_err(int offset, int err_type)
{ … }

/* Split symbol and offset. */
int traceprobe_split_symbol_offset(char *symbol, long *offset)
{ … }

/* @buf must has MAX_EVENT_NAME_LEN size */
int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
				char *buf, int offset)
{ … }

static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
				 struct traceprobe_parse_context *ctx)
{ … }

#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS

static u32 btf_type_int(const struct btf_type *t)
{
	return *(u32 *)(t + 1);
}

static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
{
	const struct btf_type *real_type;
	u32 intdata;
	s32 tid;

	real_type = btf_type_skip_modifiers(btf, type->type, &tid);
	if (!real_type)
		return false;

	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
		return false;

	intdata = btf_type_int(real_type);
	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
		&& BTF_INT_BITS(intdata) == 8;
}

static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
{
	const struct btf_type *real_type;
	const struct btf_array *array;
	u32 intdata;
	s32 tid;

	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
		return false;

	array = (const struct btf_array *)(type + 1);

	real_type = btf_type_skip_modifiers(btf, array->type, &tid);

	intdata = btf_type_int(real_type);
	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
		&& BTF_INT_BITS(intdata) == 8;
}

static int check_prepare_btf_string_fetch(char *typename,
				struct fetch_insn **pcode,
				struct traceprobe_parse_context *ctx)
{
	struct btf *btf = ctx->btf;

	if (!btf || !ctx->last_type)
		return 0;

	/* char [] does not need any change. */
	if (btf_type_is_char_array(btf, ctx->last_type))
		return 0;

	/* char * requires dereference the pointer. */
	if (btf_type_is_char_ptr(btf, ctx->last_type)) {
		struct fetch_insn *code = *pcode + 1;

		if (code->op == FETCH_OP_END) {
			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
			return -E2BIG;
		}
		if (typename[0] == 'u')
			code->op = FETCH_OP_UDEREF;
		else
			code->op = FETCH_OP_DEREF;
		code->offset = 0;
		*pcode = code;
		return 0;
	}
	/* Other types are not available for string */
	trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
	return -EINVAL;
}

static const char *fetch_type_from_btf_type(struct btf *btf,
					const struct btf_type *type,
					struct traceprobe_parse_context *ctx)
{
	u32 intdata;

	/* TODO: const char * could be converted as a string */
	switch (BTF_INFO_KIND(type->info)) {
	case BTF_KIND_ENUM:
		/* enum is "int", so convert to "s32" */
		return "s32";
	case BTF_KIND_ENUM64:
		return "s64";
	case BTF_KIND_PTR:
		/* pointer will be converted to "x??" */
		if (IS_ENABLED(CONFIG_64BIT))
			return "x64";
		else
			return "x32";
	case BTF_KIND_INT:
		intdata = btf_type_int(type);
		if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
			switch (BTF_INT_BITS(intdata)) {
			case 8:
				return "s8";
			case 16:
				return "s16";
			case 32:
				return "s32";
			case 64:
				return "s64";
			}
		} else {	/* unsigned */
			switch (BTF_INT_BITS(intdata)) {
			case 8:
				return "u8";
			case 16:
				return "u16";
			case 32:
				return "u32";
			case 64:
				return "u64";
			}
			/* bitfield, size is encoded in the type */
			ctx->last_bitsize = BTF_INT_BITS(intdata);
			ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
			return "u64";
		}
	}
	/* TODO: support other types */

	return NULL;
}

static int query_btf_context(struct traceprobe_parse_context *ctx)
{
	const struct btf_param *param;
	const struct btf_type *type;
	struct btf *btf;
	s32 nr;

	if (ctx->btf)
		return 0;

	if (!ctx->funcname)
		return -EINVAL;

	type = btf_find_func_proto(ctx->funcname, &btf);
	if (!type)
		return -ENOENT;

	ctx->btf = btf;
	ctx->proto = type;

	/* ctx->params is optional, since func(void) will not have params. */
	nr = 0;
	param = btf_get_func_param(type, &nr);
	if (!IS_ERR_OR_NULL(param)) {
		/* Hide the first 'data' argument of tracepoint */
		if (ctx->flags & TPARG_FL_TPOINT) {
			nr--;
			param++;
		}
	}

	if (nr > 0) {
		ctx->nr_params = nr;
		ctx->params = param;
	} else {
		ctx->nr_params = 0;
		ctx->params = NULL;
	}

	return 0;
}

static void clear_btf_context(struct traceprobe_parse_context *ctx)
{
	if (ctx->btf) {
		btf_put(ctx->btf);
		ctx->btf = NULL;
		ctx->proto = NULL;
		ctx->params = NULL;
		ctx->nr_params = 0;
	}
}

/* Return 1 if the field separater is arrow operator ('->') */
static int split_next_field(char *varname, char **next_field,
			    struct traceprobe_parse_context *ctx)
{
	char *field;
	int ret = 0;

	field = strpbrk(varname, ".-");
	if (field) {
		if (field[0] == '-' && field[1] == '>') {
			field[0] = '\0';
			field += 2;
			ret = 1;
		} else if (field[0] == '.') {
			field[0] = '\0';
			field += 1;
		} else {
			trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
			return -EINVAL;
		}
		*next_field = field;
	}

	return ret;
}

/*
 * Parse the field of data structure. The @type must be a pointer type
 * pointing the target data structure type.
 */
static int parse_btf_field(char *fieldname, const struct btf_type *type,
			   struct fetch_insn **pcode, struct fetch_insn *end,
			   struct traceprobe_parse_context *ctx)
{
	struct fetch_insn *code = *pcode;
	const struct btf_member *field;
	u32 bitoffs, anon_offs;
	char *next;
	int is_ptr;
	s32 tid;

	do {
		/* Outer loop for solving arrow operator ('->') */
		if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
			trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
			return -EINVAL;
		}
		/* Convert a struct pointer type to a struct type */
		type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
		if (!type) {
			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
			return -EINVAL;
		}

		bitoffs = 0;
		do {
			/* Inner loop for solving dot operator ('.') */
			next = NULL;
			is_ptr = split_next_field(fieldname, &next, ctx);
			if (is_ptr < 0)
				return is_ptr;

			anon_offs = 0;
			field = btf_find_struct_member(ctx->btf, type, fieldname,
						       &anon_offs);
			if (IS_ERR(field)) {
				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
				return PTR_ERR(field);
			}
			if (!field) {
				trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
				return -ENOENT;
			}
			/* Add anonymous structure/union offset */
			bitoffs += anon_offs;

			/* Accumulate the bit-offsets of the dot-connected fields */
			if (btf_type_kflag(type)) {
				bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
				ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
			} else {
				bitoffs += field->offset;
				ctx->last_bitsize = 0;
			}

			type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
			if (!type) {
				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
				return -EINVAL;
			}

			ctx->offset += next - fieldname;
			fieldname = next;
		} while (!is_ptr && fieldname);

		if (++code == end) {
			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
			return -EINVAL;
		}
		code->op = FETCH_OP_DEREF;	/* TODO: user deref support */
		code->offset = bitoffs / 8;
		*pcode = code;

		ctx->last_bitoffs = bitoffs % 8;
		ctx->last_type = type;
	} while (fieldname);

	return 0;
}

static int __store_entry_arg(struct trace_probe *tp, int argnum);

static int parse_btf_arg(char *varname,
			 struct fetch_insn **pcode, struct fetch_insn *end,
			 struct traceprobe_parse_context *ctx)
{
	struct fetch_insn *code = *pcode;
	const struct btf_param *params;
	const struct btf_type *type;
	char *field = NULL;
	int i, is_ptr, ret;
	u32 tid;

	if (WARN_ON_ONCE(!ctx->funcname))
		return -EINVAL;

	is_ptr = split_next_field(varname, &field, ctx);
	if (is_ptr < 0)
		return is_ptr;
	if (!is_ptr && field) {
		/* dot-connected field on an argument is not supported. */
		trace_probe_log_err(ctx->offset + field - varname,
				    NOSUP_DAT_ARG);
		return -EOPNOTSUPP;
	}

	if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
		code->op = FETCH_OP_RETVAL;
		/* Check whether the function return type is not void */
		if (query_btf_context(ctx) == 0) {
			if (ctx->proto->type == 0) {
				trace_probe_log_err(ctx->offset, NO_RETVAL);
				return -ENOENT;
			}
			tid = ctx->proto->type;
			goto found;
		}
		if (field) {
			trace_probe_log_err(ctx->offset + field - varname,
					    NO_BTF_ENTRY);
			return -ENOENT;
		}
		return 0;
	}

	if (!ctx->btf) {
		ret = query_btf_context(ctx);
		if (ret < 0 || ctx->nr_params == 0) {
			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
			return PTR_ERR(params);
		}
	}
	params = ctx->params;

	for (i = 0; i < ctx->nr_params; i++) {
		const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);

		if (name && !strcmp(name, varname)) {
			if (tparg_is_function_entry(ctx->flags)) {
				code->op = FETCH_OP_ARG;
				if (ctx->flags & TPARG_FL_TPOINT)
					code->param = i + 1;
				else
					code->param = i;
			} else if (tparg_is_function_return(ctx->flags)) {
				code->op = FETCH_OP_EDATA;
				ret = __store_entry_arg(ctx->tp, i);
				if (ret < 0) {
					/* internal error */
					return ret;
				}
				code->offset = ret;
			}
			tid = params[i].type;
			goto found;
		}
	}
	trace_probe_log_err(ctx->offset, NO_BTFARG);
	return -ENOENT;

found:
	type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
	if (!type) {
		trace_probe_log_err(ctx->offset, BAD_BTF_TID);
		return -EINVAL;
	}
	/* Initialize the last type information */
	ctx->last_type = type;
	ctx->last_bitoffs = 0;
	ctx->last_bitsize = 0;
	if (field) {
		ctx->offset += field - varname;
		return parse_btf_field(field, type, pcode, end, ctx);
	}
	return 0;
}

static const struct fetch_type *find_fetch_type_from_btf_type(
					struct traceprobe_parse_context *ctx)
{
	struct btf *btf = ctx->btf;
	const char *typestr = NULL;

	if (btf && ctx->last_type)
		typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);

	return find_fetch_type(typestr, ctx->flags);
}

static int parse_btf_bitfield(struct fetch_insn **pcode,
			      struct traceprobe_parse_context *ctx)
{
	struct fetch_insn *code = *pcode;

	if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
		return 0;

	code++;
	if (code->op != FETCH_OP_NOP) {
		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
		return -EINVAL;
	}
	*pcode = code;

	code->op = FETCH_OP_MOD_BF;
	code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
	code->rshift = 64 - ctx->last_bitsize;
	code->basesize = 64 / 8;
	return 0;
}

#else
static void clear_btf_context(struct traceprobe_parse_context *ctx)
{ … }

static int query_btf_context(struct traceprobe_parse_context *ctx)
{ … }

static int parse_btf_arg(char *varname,
			 struct fetch_insn **pcode, struct fetch_insn *end,
			 struct traceprobe_parse_context *ctx)
{ … }

static int parse_btf_bitfield(struct fetch_insn **pcode,
			      struct traceprobe_parse_context *ctx)
{ … }

#define find_fetch_type_from_btf_type(ctx) …

static int check_prepare_btf_string_fetch(char *typename,
				struct fetch_insn **pcode,
				struct traceprobe_parse_context *ctx)
{ … }

#endif

#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API

static int __store_entry_arg(struct trace_probe *tp, int argnum)
{ … }

int traceprobe_get_entry_data_size(struct trace_probe *tp)
{ … }

void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
{ … }
NOKPROBE_SYMBOL(…)
#endif

#define PARAM_MAX_STACK …

/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
			    struct fetch_insn **pcode,
			    struct fetch_insn *end,
			    struct traceprobe_parse_context *ctx)
{ … }

static int str_to_immediate(char *str, unsigned long *imm)
{ … }

static int __parse_imm_string(char *str, char **pbuf, int offs)
{ … }

/* Recursive argument parser */
static int
parse_probe_arg(char *arg, const struct fetch_type *type,
		struct fetch_insn **pcode, struct fetch_insn *end,
		struct traceprobe_parse_context *ctx)
{ … }

/* Bitfield type needs to be parsed into a fetch function */
static int __parse_bitfield_probe_arg(const char *bf,
				      const struct fetch_type *t,
				      struct fetch_insn **pcode)
{ … }

/* Split type part from @arg and return it. */
static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
				  struct traceprobe_parse_context *ctx)
{ … }

/* After parsing, adjust the fetch_insn according to the probe_arg */
static int finalize_fetch_insn(struct fetch_insn *code,
			       struct probe_arg *parg,
			       char *type,
			       int type_offset,
			       struct traceprobe_parse_context *ctx)
{ … }

/* String length checking wrapper */
static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
					   struct probe_arg *parg,
					   struct traceprobe_parse_context *ctx)
{ … }

/* Return 1 if name is reserved or already used by another argument */
static int traceprobe_conflict_field_name(const char *name,
					  struct probe_arg *args, int narg)
{ … }

static char *generate_probe_arg_name(const char *arg, int idx)
{ … }

int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
			       struct traceprobe_parse_context *ctx)
{ … }

void traceprobe_free_probe_arg(struct probe_arg *arg)
{ … }

static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
			    struct traceprobe_parse_context *ctx)
{ … }

static int sprint_nth_btf_arg(int idx, const char *type,
			      char *buf, int bufsize,
			      struct traceprobe_parse_context *ctx)
{ … }

/* Return new_argv which must be freed after use */
const char **traceprobe_expand_meta_args(int argc, const char *argv[],
					 int *new_argc, char *buf, int bufsize,
					 struct traceprobe_parse_context *ctx)
{ … }

/* @buf: *buf must be equal to NULL. Caller must to free *buf */
int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf)
{ … }

void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
{ … }

int traceprobe_update_arg(struct probe_arg *arg)
{ … }

/* When len=0, we just calculate the needed length */
#define LEN_OR_ZERO …
static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
			   enum probe_print_type ptype)
{ … }
#undef LEN_OR_ZERO

int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
{ … }

int traceprobe_define_arg_fields(struct trace_event_call *event_call,
				 size_t offset, struct trace_probe *tp)
{ … }

static void trace_probe_event_free(struct trace_probe_event *tpe)
{ … }

int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
{ … }

void trace_probe_unlink(struct trace_probe *tp)
{ … }

void trace_probe_cleanup(struct trace_probe *tp)
{ … }

int trace_probe_init(struct trace_probe *tp, const char *event,
		     const char *group, bool alloc_filter, int nargs)
{ … }

static struct trace_event_call *
find_trace_event_call(const char *system, const char *event_name)
{ … }

int trace_probe_register_event_call(struct trace_probe *tp)
{ … }

int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
{ … }

struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
						  struct trace_event_file *file)
{ … }

int trace_probe_remove_file(struct trace_probe *tp,
			    struct trace_event_file *file)
{ … }

/*
 * Return the smallest index of different type argument (start from 1).
 * If all argument types and name are same, return 0.
 */
int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
{ … }

bool trace_probe_match_command_args(struct trace_probe *tp,
				    int argc, const char **argv)
{ … }

int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
{ … }

int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
		 u8 *data, void *field)
{ … }