linux/arch/x86/kernel/callthunks.c

// SPDX-License-Identifier: GPL-2.0-only

#define pr_fmt(fmt) …

#include <linux/debugfs.h>
#include <linux/kallsyms.h>
#include <linux/memory.h>
#include <linux/moduleloader.h>
#include <linux/static_call.h>

#include <asm/alternative.h>
#include <asm/asm-offsets.h>
#include <asm/cpu.h>
#include <asm/ftrace.h>
#include <asm/insn.h>
#include <asm/kexec.h>
#include <asm/nospec-branch.h>
#include <asm/paravirt.h>
#include <asm/sections.h>
#include <asm/switch_to.h>
#include <asm/sync_core.h>
#include <asm/text-patching.h>
#include <asm/xen/hypercall.h>

static int __initdata_or_module debug_callthunks;

#define MAX_PATCH_LEN …

#define prdbg(fmt, args...) …

static int __init debug_thunks(char *str)
{ … }
__setup(…);

#ifdef CONFIG_CALL_THUNKS_DEBUG
DEFINE_PER_CPU(u64, __x86_call_count);
DEFINE_PER_CPU(u64, __x86_ret_count);
DEFINE_PER_CPU(u64, __x86_stuffs_count);
DEFINE_PER_CPU(u64, __x86_ctxsw_count);
EXPORT_PER_CPU_SYMBOL_GPL(…);
EXPORT_PER_CPU_SYMBOL_GPL(…);
#endif

extern s32 __call_sites[], __call_sites_end[];

struct core_text { … };

static bool thunks_initialized __ro_after_init;

static const struct core_text builtin_coretext = …;

asm …;

extern u8 skl_call_thunk_template[];
extern u8 skl_call_thunk_tail[];

#define SKL_TMPL_SIZE …

extern void error_entry(void);
extern void xen_error_entry(void);
extern void paranoid_entry(void);

static inline bool within_coretext(const struct core_text *ct, void *addr)
{ … }

static inline bool within_module_coretext(void *addr)
{ … }

static bool is_coretext(const struct core_text *ct, void *addr)
{ … }

static bool skip_addr(void *dest)
{ … }

static __init_or_module void *call_get_dest(void *addr)
{ … }

static const u8 nops[] = …;

static void *patch_dest(void *dest, bool direct)
{ … }

static __init_or_module void patch_call(void *addr, const struct core_text *ct)
{ … }

static __init_or_module void
patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
{ … }

static __init_or_module void
patch_alt_call_sites(struct alt_instr *start, struct alt_instr *end,
		     const struct core_text *ct)
{ … }

static __init_or_module void
callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
{ … }

void __init callthunks_patch_builtin_calls(void)
{ … }

void *callthunks_translate_call_dest(void *dest)
{ … }

#ifdef CONFIG_BPF_JIT
static bool is_callthunk(void *addr)
{ … }

int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip)
{ … }
#endif

#ifdef CONFIG_MODULES
void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
					    struct module *mod)
{ … }
#endif /* CONFIG_MODULES */

#if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
static int callthunks_debug_show(struct seq_file *m, void *p)
{ … }

static int callthunks_debug_open(struct inode *inode, struct file *file)
{ … }

static const struct file_operations dfs_ops = …;

static int __init callthunks_debugfs_init(void)
{ … }
__initcall(callthunks_debugfs_init);
#endif