/* * Performance events - AMD IBS * * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter * * For licencing details see kernel-base/COPYING */ #include <linux/perf_event.h> #include <linux/init.h> #include <linux/export.h> #include <linux/pci.h> #include <linux/ptrace.h> #include <linux/syscore_ops.h> #include <linux/sched/clock.h> #include <asm/apic.h> #include "../perf_event.h" static u32 ibs_caps; #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) #include <linux/kprobes.h> #include <linux/hardirq.h> #include <asm/nmi.h> #include <asm/amd-ibs.h> #define IBS_FETCH_CONFIG_MASK … #define IBS_OP_CONFIG_MASK … /* * IBS states: * * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken * and any further add()s must fail. * * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are * complicated by the fact that the IBS hardware can send late NMIs (ie. after * we've cleared the EN bit). * * In order to consume these late NMIs we have the STOPPED state, any NMI that * happens after we've cleared the EN state will clear this bit and report the * NMI handled (this is fundamentally racy in the face or multiple NMI sources, * someone else can consume our BIT and our NMI will go unhandled). * * And since we cannot set/clear this separate bit together with the EN bit, * there are races; if we cleared STARTED early, an NMI could land in * between clearing STARTED and clearing the EN bit (in fact multiple NMIs * could happen if the period is small enough), and consume our STOPPED bit * and trigger streams of unhandled NMIs. * * If, however, we clear STARTED late, an NMI can hit between clearing the * EN bit and clearing STARTED, still see STARTED set and process the event. * If this event will have the VALID bit clear, we bail properly, but this * is not a given. With VALID set we can end up calling pmu::stop() again * (the throttle logic) and trigger the WARNs in there. * * So what we do is set STOPPING before clearing EN to avoid the pmu::stop() * nesting, and clear STARTED late, so that we have a well defined state over * the clearing of the EN bit. * * XXX: we could probably be using !atomic bitops for all this. */ enum ibs_states { … }; struct cpu_perf_ibs { … }; struct perf_ibs { … }; static int perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) { … } static int perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) { … } static struct perf_ibs perf_ibs_fetch; static struct perf_ibs perf_ibs_op; static struct perf_ibs *get_ibs_pmu(int type) { … } /* * core pmu config -> IBS config * * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count * perf record -a -e r076:p ... # same as -e cpu-cycles:p * perf record -a -e r0C1:p ... # use ibs op counting micro-ops * * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, * MSRC001_1033) is used to select either cycle or micro-ops counting * mode. */ static int core_pmu_ibs_config(struct perf_event *event, u64 *config) { … } /* * The rip of IBS samples has skid 0. Thus, IBS supports precise * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the * rip is invalid when IBS was not able to record the rip correctly. * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. */ int forward_event_to_ibs(struct perf_event *event) { … } /* * Grouping of IBS events is not possible since IBS can have only * one event active at any point in time. */ static int validate_group(struct perf_event *event) { … } static int perf_ibs_init(struct perf_event *event) { … } static int perf_ibs_set_period(struct perf_ibs *perf_ibs, struct hw_perf_event *hwc, u64 *period) { … } static u64 get_ibs_fetch_count(u64 config) { … } static u64 get_ibs_op_count(u64 config) { … } static void perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, u64 *config) { … } static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, struct hw_perf_event *hwc, u64 config) { … } /* * Erratum #420 Instruction-Based Sampling Engine May Generate * Interrupt that Cannot Be Cleared: * * Must clear counter mask first, then clear the enable bit. See * Revision Guide for AMD Family 10h Processors, Publication #41322. */ static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, struct hw_perf_event *hwc, u64 config) { … } /* * We cannot restore the ibs pmu state, so we always needs to update * the event while stopping it and then reset the state when starting * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in * perf_ibs_start()/perf_ibs_stop() and instead always do it. */ static void perf_ibs_start(struct perf_event *event, int flags) { … } static void perf_ibs_stop(struct perf_event *event, int flags) { … } static int perf_ibs_add(struct perf_event *event, int flags) { … } static void perf_ibs_del(struct perf_event *event, int flags) { … } static void perf_ibs_read(struct perf_event *event) { … } /* * We need to initialize with empty group if all attributes in the * group are dynamic. */ static struct attribute *attrs_empty[] = …; static struct attribute_group empty_format_group = …; static struct attribute_group empty_caps_group = …; static const struct attribute_group *empty_attr_groups[] = …; PMU_FORMAT_ATTR(…); PMU_FORMAT_ATTR(…); PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); static umode_t zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) { … } static struct attribute *rand_en_attrs[] = …; static struct attribute *fetch_l3missonly_attrs[] = …; static struct attribute *zen4_ibs_extensions_attrs[] = …; static struct attribute_group group_rand_en = …; static struct attribute_group group_fetch_l3missonly = …; static struct attribute_group group_zen4_ibs_extensions = …; static const struct attribute_group *fetch_attr_groups[] = …; static const struct attribute_group *fetch_attr_update[] = …; static umode_t cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i) { … } static struct attribute *cnt_ctl_attrs[] = …; static struct attribute *op_l3missonly_attrs[] = …; static struct attribute_group group_cnt_ctl = …; static struct attribute_group group_op_l3missonly = …; static const struct attribute_group *op_attr_update[] = …; static struct perf_ibs perf_ibs_fetch = …; static struct perf_ibs perf_ibs_op = …; static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3, struct perf_sample_data *data) { … } /* * Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has * more fine granular DataSrc encodings. Others have coarse. */ static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2) { … } #define L(x) … #define LN(x) … #define REM … #define HOPS(x) … static u64 g_data_src[8] = …; #define RMT_NODE_BITS … #define RMT_NODE_APPLICABLE(x) … static u64 g_zen4_data_src[32] = …; #define ZEN4_RMT_NODE_BITS … #define ZEN4_RMT_NODE_APPLICABLE(x) … static __u64 perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, union ibs_op_data3 *op_data3, struct perf_sample_data *data) { … } static bool perf_ibs_cache_hit_st_valid(void) { … } static void perf_ibs_get_mem_snoop(union ibs_op_data2 *op_data2, struct perf_sample_data *data) { … } static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, struct perf_sample_data *data) { … } static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3, struct perf_sample_data *data) { … } #define ibs_op_msr_idx(msr) … static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, struct perf_sample_data *data, union ibs_op_data2 *op_data2, union ibs_op_data3 *op_data3) { … } static __u64 perf_ibs_get_op_data2(struct perf_ibs_data *ibs_data, union ibs_op_data3 *op_data3) { … } static void perf_ibs_parse_ld_st_data(__u64 sample_type, struct perf_ibs_data *ibs_data, struct perf_sample_data *data) { … } static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type, int check_rip) { … } static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) { … } static int perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) { … } NOKPROBE_SYMBOL(perf_ibs_nmi_handler); static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) { … } static __init int perf_ibs_fetch_init(void) { … } static __init int perf_ibs_op_init(void) { … } static __init int perf_event_ibs_init(void) { … } #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ static __init int perf_event_ibs_init(void) { return 0; } #endif /* IBS - apic initialization, for perf and oprofile */ static __init u32 __get_ibs_caps(void) { … } u32 get_ibs_caps(void) { … } EXPORT_SYMBOL(…); static inline int get_eilvt(int offset) { … } static inline int put_eilvt(int offset) { … } /* * Check and reserve APIC extended interrupt LVT offset for IBS if available. */ static inline int ibs_eilvt_valid(void) { … } static int setup_ibs_ctl(int ibs_eilvt_off) { … } /* * This runs only on the current cpu. We try to find an LVT offset and * setup the local APIC. For this we must disable preemption. On * success we initialize all nodes with this offset. This updates then * the offset in the IBS_CTL per-node msr. The per-core APIC setup of * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that * is using the new offset. */ static void force_ibs_eilvt_setup(void) { … } static void ibs_eilvt_setup(void) { … } static inline int get_ibs_lvt_offset(void) { … } static void setup_APIC_ibs(void) { … } static void clear_APIC_ibs(void) { … } static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu) { … } #ifdef CONFIG_PM static int perf_ibs_suspend(void) { … } static void perf_ibs_resume(void) { … } static struct syscore_ops perf_ibs_syscore_ops = …; static void perf_ibs_pm_init(void) { … } #else static inline void perf_ibs_pm_init(void) { } #endif static int x86_pmu_amd_ibs_dying_cpu(unsigned int cpu) { … } static __init int amd_ibs_init(void) { … } /* Since we need the pci subsystem to init ibs we can't do this earlier: */ device_initcall(amd_ibs_init);
Codebase Browser
linux