linux/arch/x86/kernel/unwind_orc.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/objtool.h>
#include <linux/module.h>
#include <linux/sort.h>
#include <asm/ptrace.h>
#include <asm/stacktrace.h>
#include <asm/unwind.h>
#include <asm/orc_types.h>
#include <asm/orc_lookup.h>
#include <asm/orc_header.h>

ORC_HEADER;

#define orc_warn(fmt, ...) …

#define orc_warn_current(args...) …

extern int __start_orc_unwind_ip[];
extern int __stop_orc_unwind_ip[];
extern struct orc_entry __start_orc_unwind[];
extern struct orc_entry __stop_orc_unwind[];

static bool orc_init __ro_after_init;
static bool unwind_debug __ro_after_init;
static unsigned int lookup_num_blocks __ro_after_init;

static int __init unwind_debug_cmdline(char *str)
{ … }
early_param(…);

static void unwind_dump(struct unwind_state *state)
{ … }

static inline unsigned long orc_ip(const int *ip)
{ … }

static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
				    unsigned int num_entries, unsigned long ip)
{ … }

#ifdef CONFIG_MODULES
static struct orc_entry *orc_module_find(unsigned long ip)
{ … }
#else
static struct orc_entry *orc_module_find(unsigned long ip)
{
	return NULL;
}
#endif

#ifdef CONFIG_DYNAMIC_FTRACE
static struct orc_entry *orc_find(unsigned long ip);

/*
 * Ftrace dynamic trampolines do not have orc entries of their own.
 * But they are copies of the ftrace entries that are static and
 * defined in ftrace_*.S, which do have orc entries.
 *
 * If the unwinder comes across a ftrace trampoline, then find the
 * ftrace function that was used to create it, and use that ftrace
 * function's orc entry, as the placement of the return code in
 * the stack will be identical.
 */
static struct orc_entry *orc_ftrace_find(unsigned long ip)
{ … }
#else
static struct orc_entry *orc_ftrace_find(unsigned long ip)
{
	return NULL;
}
#endif

/*
 * If we crash with IP==0, the last successfully executed instruction
 * was probably an indirect function call with a NULL function pointer,
 * and we don't have unwind information for NULL.
 * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
 * pointer into its parent and then continue normally from there.
 */
static struct orc_entry null_orc_entry = …;

/* Fake frame pointer entry -- used as a fallback for generated code */
static struct orc_entry orc_fp_entry = …;

static struct orc_entry *orc_find(unsigned long ip)
{ … }

#ifdef CONFIG_MODULES

static DEFINE_MUTEX(sort_mutex);
static int *cur_orc_ip_table = …;
static struct orc_entry *cur_orc_table = …;

static void orc_sort_swap(void *_a, void *_b, int size)
{ … }

static int orc_sort_cmp(const void *_a, const void *_b)
{ … }

void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
			void *_orc, size_t orc_size)
{ … }
#endif

void __init unwind_init(void)
{ … }

unsigned long unwind_get_return_address(struct unwind_state *state)
{ … }
EXPORT_SYMBOL_GPL(…);

unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)
{ … }

static bool stack_access_ok(struct unwind_state *state, unsigned long _addr,
			    size_t len)
{ … }

static bool deref_stack_reg(struct unwind_state *state, unsigned long addr,
			    unsigned long *val)
{ … }

static bool deref_stack_regs(struct unwind_state *state, unsigned long addr,
			     unsigned long *ip, unsigned long *sp)
{ … }

static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr,
				  unsigned long *ip, unsigned long *sp)
{ … }

/*
 * If state->regs is non-NULL, and points to a full pt_regs, just get the reg
 * value from state->regs.
 *
 * Otherwise, if state->regs just points to IRET regs, and the previous frame
 * had full regs, it's safe to get the value from the previous regs.  This can
 * happen when early/late IRQ entry code gets interrupted by an NMI.
 */
static bool get_reg(struct unwind_state *state, unsigned int reg_off,
		    unsigned long *val)
{ … }

bool unwind_next_frame(struct unwind_state *state)
{ … }
EXPORT_SYMBOL_GPL(…);

void __unwind_start(struct unwind_state *state, struct task_struct *task,
		    struct pt_regs *regs, unsigned long *first_frame)
{ … }
EXPORT_SYMBOL_GPL(…);