// SPDX-License-Identifier: GPL-2.0 #include <linux/perf_event.h> #include <linux/types.h> #include <asm/cpu_device_id.h> #include <asm/perf_event.h> #include <asm/msr.h> #include "../perf_event.h" /* * Intel LBR_SELECT bits * Intel Vol3a, April 2011, Section 16.7 Table 16-10 * * Hardware branch filter (not available on all CPUs) */ #define LBR_KERNEL_BIT … #define LBR_USER_BIT … #define LBR_JCC_BIT … #define LBR_REL_CALL_BIT … #define LBR_IND_CALL_BIT … #define LBR_RETURN_BIT … #define LBR_IND_JMP_BIT … #define LBR_REL_JMP_BIT … #define LBR_FAR_BIT … #define LBR_CALL_STACK_BIT … /* * Following bit only exists in Linux; we mask it out before writing it to * the actual MSR. But it helps the constraint perf code to understand * that this is a separate configuration. */ #define LBR_NO_INFO_BIT … #define LBR_KERNEL … #define LBR_USER … #define LBR_JCC … #define LBR_REL_CALL … #define LBR_IND_CALL … #define LBR_RETURN … #define LBR_REL_JMP … #define LBR_IND_JMP … #define LBR_FAR … #define LBR_CALL_STACK … #define LBR_NO_INFO … #define LBR_PLM … #define LBR_SEL_MASK … #define LBR_NOT_SUPP … #define LBR_IGN … #define LBR_ANY … #define LBR_FROM_FLAG_MISPRED … #define LBR_FROM_FLAG_IN_TX … #define LBR_FROM_FLAG_ABORT … #define LBR_FROM_SIGNEXT_2MSB … /* * Intel LBR_CTL bits * * Hardware branch filter for Arch LBR */ #define ARCH_LBR_KERNEL_BIT … #define ARCH_LBR_USER_BIT … #define ARCH_LBR_CALL_STACK_BIT … #define ARCH_LBR_JCC_BIT … #define ARCH_LBR_REL_JMP_BIT … #define ARCH_LBR_IND_JMP_BIT … #define ARCH_LBR_REL_CALL_BIT … #define ARCH_LBR_IND_CALL_BIT … #define ARCH_LBR_RETURN_BIT … #define ARCH_LBR_OTHER_BRANCH_BIT … #define ARCH_LBR_KERNEL … #define ARCH_LBR_USER … #define ARCH_LBR_CALL_STACK … #define ARCH_LBR_JCC … #define ARCH_LBR_REL_JMP … #define ARCH_LBR_IND_JMP … #define ARCH_LBR_REL_CALL … #define ARCH_LBR_IND_CALL … #define ARCH_LBR_RETURN … #define ARCH_LBR_OTHER_BRANCH … #define ARCH_LBR_ANY … #define ARCH_LBR_CTL_MASK … static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); static __always_inline bool is_lbr_call_stack_bit_set(u64 config) { … } /* * We only support LBR implementations that have FREEZE_LBRS_ON_PMI * otherwise it becomes near impossible to get a reliable stack. */ static void __intel_pmu_lbr_enable(bool pmi) { … } void intel_pmu_lbr_reset_32(void) { … } void intel_pmu_lbr_reset_64(void) { … } static void intel_pmu_arch_lbr_reset(void) { … } void intel_pmu_lbr_reset(void) { … } /* * TOS = most recently recorded branch */ static inline u64 intel_pmu_lbr_tos(void) { … } enum { … }; /* * For format LBR_FORMAT_EIP_FLAGS2, bits 61:62 in MSR_LAST_BRANCH_FROM_x * are the TSX flags when TSX is supported, but when TSX is not supported * they have no consistent behavior: * * - For wrmsr(), bits 61:62 are considered part of the sign extension. * - For HW updates (branch captures) bits 61:62 are always OFF and are not * part of the sign extension. * * Therefore, if: * * 1) LBR format LBR_FORMAT_EIP_FLAGS2 * 2) CPU has no TSX support enabled * * ... then any value passed to wrmsr() must be sign extended to 63 bits and any * value from rdmsr() must be converted to have a 61 bits sign extension, * ignoring the TSX flags. */ static inline bool lbr_from_signext_quirk_needed(void) { … } static DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); /* If quirk is enabled, ensure sign extension is 63 bits: */ inline u64 lbr_from_signext_quirk_wr(u64 val) { … } /* * If quirk is needed, ensure sign extension is 61 bits: */ static u64 lbr_from_signext_quirk_rd(u64 val) { … } static __always_inline void wrlbr_from(unsigned int idx, u64 val) { … } static __always_inline void wrlbr_to(unsigned int idx, u64 val) { … } static __always_inline void wrlbr_info(unsigned int idx, u64 val) { … } static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr) { … } static __always_inline u64 rdlbr_to(unsigned int idx, struct lbr_entry *lbr) { … } static __always_inline u64 rdlbr_info(unsigned int idx, struct lbr_entry *lbr) { … } static inline void wrlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) { … } static inline bool rdlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) { … } void intel_pmu_lbr_restore(void *ctx) { … } static void intel_pmu_arch_lbr_restore(void *ctx) { … } /* * Restore the Architecture LBR state from the xsave area in the perf * context data for the task via the XRSTORS instruction. */ static void intel_pmu_arch_lbr_xrstors(void *ctx) { … } static __always_inline bool lbr_is_reset_in_cstate(void *ctx) { … } static void __intel_pmu_lbr_restore(void *ctx) { … } void intel_pmu_lbr_save(void *ctx) { … } static void intel_pmu_arch_lbr_save(void *ctx) { … } /* * Save the Architecture LBR state to the xsave area in the perf * context data for the task via the XSAVES instruction. */ static void intel_pmu_arch_lbr_xsaves(void *ctx) { … } static void __intel_pmu_lbr_save(void *ctx) { … } void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc, struct perf_event_pmu_context *next_epc) { … } void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) { … } static inline bool branch_user_callstack(unsigned br_sel) { … } void intel_pmu_lbr_add(struct perf_event *event) { … } void release_lbr_buffers(void) { … } void reserve_lbr_buffers(void) { … } void intel_pmu_lbr_del(struct perf_event *event) { … } static inline bool vlbr_exclude_host(void) { … } void intel_pmu_lbr_enable_all(bool pmi) { … } void intel_pmu_lbr_disable_all(void) { … } void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) { … } /* * Due to lack of segmentation in Linux the effective address (offset) * is the same as the linear address, allowing us to merge the LIP and EIP * LBR formats. */ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) { … } static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred); static DEFINE_STATIC_KEY_FALSE(x86_lbr_cycles); static DEFINE_STATIC_KEY_FALSE(x86_lbr_type); static __always_inline int get_lbr_br_type(u64 info) { … } static __always_inline bool get_lbr_mispred(u64 info) { … } static __always_inline u16 get_lbr_cycles(u64 info) { … } static_assert(…); static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, struct lbr_entry *entries) { … } /* * The enabled order may be different from the counter order. * Update the lbr_counters with the enabled order. */ static void intel_pmu_lbr_counters_reorder(struct cpu_hw_events *cpuc, struct perf_event *event) { … } void intel_pmu_lbr_save_brstack(struct perf_sample_data *data, struct cpu_hw_events *cpuc, struct perf_event *event) { … } static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc) { … } static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc) { … } void intel_pmu_lbr_read(void) { … } /* * SW filter is used: * - in case there is no HW filter * - in case the HW filter has errata or limitations */ static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event) { … } /* * setup the HW LBR filter * Used only when available, may not be enough to disambiguate * all branches, may need the help of the SW filter */ static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) { … } int intel_pmu_setup_lbr_filter(struct perf_event *event) { … } enum { … }; static const int arch_lbr_br_type_map[ARCH_LBR_BR_TYPE_MAP_MAX] = …; /* * implement actual branch filter based on user demand. * Hardware may not exactly satisfy that request, thus * we need to inspect opcodes. Mismatched branches are * discarded. Therefore, the number of branches returned * in PERF_SAMPLE_BRANCH_STACK sample may vary. */ static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) { … } void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr) { … } /* * Map interface branch filters onto LBR filters */ static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = …; static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = …; static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = …; static int arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = …; /* core */ void __init intel_pmu_lbr_init_core(void) { … } /* nehalem/westmere */ void __init intel_pmu_lbr_init_nhm(void) { … } /* sandy bridge */ void __init intel_pmu_lbr_init_snb(void) { … } static inline struct kmem_cache * create_lbr_kmem_cache(size_t size, size_t align) { … } /* haswell */ void intel_pmu_lbr_init_hsw(void) { … } /* skylake */ __init void intel_pmu_lbr_init_skl(void) { … } /* atom */ void __init intel_pmu_lbr_init_atom(void) { … } /* slm */ void __init intel_pmu_lbr_init_slm(void) { … } /* Knights Landing */ void intel_pmu_lbr_init_knl(void) { … } void intel_pmu_lbr_init(void) { … } /* * LBR state size is variable based on the max number of registers. * This calculates the expected state size, which should match * what the hardware enumerates for the size of XFEATURE_LBR. */ static inline unsigned int get_lbr_state_size(void) { … } static bool is_arch_lbr_xsave_available(void) { … } void __init intel_pmu_arch_lbr_init(void) { … } /** * x86_perf_get_lbr - get the LBR records information * * @lbr: the caller's memory to store the LBR records information */ void x86_perf_get_lbr(struct x86_pmu_lbr *lbr) { … } EXPORT_SYMBOL_GPL(…); struct event_constraint vlbr_constraint = …;
Codebase Browser
linux