linux/arch/x86/events/intel/uncore.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/module.h>

#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>
#include "uncore.h"
#include "uncore_discovery.h"

static bool uncore_no_discover;
module_param(uncore_no_discover, bool, 0);
MODULE_PARM_DESC(…) …;
struct intel_uncore_type *empty_uncore[] = …;
struct intel_uncore_type **uncore_msr_uncores = …;
struct intel_uncore_type **uncore_pci_uncores = …;
struct intel_uncore_type **uncore_mmio_uncores = …;

static bool pcidrv_registered;
struct pci_driver *uncore_pci_driver;
/* The PCI driver for the device which the uncore doesn't own. */
struct pci_driver *uncore_pci_sub_driver;
/* pci bus to socket mapping */
DEFINE_RAW_SPINLOCK(…);
struct list_head pci2phy_map_head = …;
struct pci_extra_dev *uncore_extra_pci_dev;
int __uncore_max_dies;

/* mask of cpus that collect uncore events */
static cpumask_t uncore_cpu_mask;

/* constraint for the fixed counter */
static struct event_constraint uncore_constraint_fixed = …;
struct event_constraint uncore_constraint_empty = …;

MODULE_DESCRIPTION(…) …;
MODULE_LICENSE(…) …;

int uncore_pcibus_to_dieid(struct pci_bus *bus)
{ … }

int uncore_die_to_segment(int die)
{ … }

int uncore_device_to_die(struct pci_dev *dev)
{ … }

static void uncore_free_pcibus_map(void)
{ … }

struct pci2phy_map *__find_pci2phy_map(int segment)
{ … }

ssize_t uncore_event_show(struct device *dev,
			  struct device_attribute *attr, char *buf)
{ … }

struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
{ … }

u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
{ … }

void uncore_mmio_exit_box(struct intel_uncore_box *box)
{ … }

u64 uncore_mmio_read_counter(struct intel_uncore_box *box,
			     struct perf_event *event)
{ … }

/*
 * generic get constraint function for shared match/mask registers.
 */
struct event_constraint *
uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
{ … }

void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
{ … }

u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
{ … }

static void uncore_assign_hw_event(struct intel_uncore_box *box,
				   struct perf_event *event, int idx)
{ … }

void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
{ … }

/*
 * The overflow interrupt is unavailable for SandyBridge-EP, is broken
 * for SandyBridge. So we use hrtimer to periodically poll the counter
 * to avoid overflow.
 */
static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
{ … }

void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
{ … }

void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
{ … }

static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
{ … }

static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
						 int node)
{ … }

/*
 * Using uncore_pmu_event_init pmu event_init callback
 * as a detection point for uncore events.
 */
static int uncore_pmu_event_init(struct perf_event *event);

static bool is_box_event(struct intel_uncore_box *box, struct perf_event *event)
{ … }

static int
uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader,
		      bool dogrp)
{ … }

static struct event_constraint *
uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
{ … }

static void uncore_put_event_constraint(struct intel_uncore_box *box,
					struct perf_event *event)
{ … }

static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
{ … }

void uncore_pmu_event_start(struct perf_event *event, int flags)
{ … }

void uncore_pmu_event_stop(struct perf_event *event, int flags)
{ … }

int uncore_pmu_event_add(struct perf_event *event, int flags)
{ … }

void uncore_pmu_event_del(struct perf_event *event, int flags)
{ … }

void uncore_pmu_event_read(struct perf_event *event)
{ … }

/*
 * validation ensures the group can be loaded onto the
 * PMU if it was the only group available.
 */
static int uncore_validate_group(struct intel_uncore_pmu *pmu,
				struct perf_event *event)
{ … }

static int uncore_pmu_event_init(struct perf_event *event)
{ … }

static void uncore_pmu_enable(struct pmu *pmu)
{ … }

static void uncore_pmu_disable(struct pmu *pmu)
{ … }

static ssize_t uncore_get_attr_cpumask(struct device *dev,
				struct device_attribute *attr, char *buf)
{ … }

static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);

static struct attribute *uncore_pmu_attrs[] = …;

static const struct attribute_group uncore_pmu_attr_group = …;

static inline int uncore_get_box_id(struct intel_uncore_type *type,
				    struct intel_uncore_pmu *pmu)
{ … }

void uncore_get_alias_name(char *pmu_name, struct intel_uncore_pmu *pmu)
{ … }

static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu)
{ … }

static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
{ … }

static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu)
{ … }

static void uncore_free_boxes(struct intel_uncore_pmu *pmu)
{ … }

static void uncore_type_exit(struct intel_uncore_type *type)
{ … }

static void uncore_types_exit(struct intel_uncore_type **types)
{ … }

static int __init uncore_type_init(struct intel_uncore_type *type, bool setid)
{ … }

static int __init
uncore_types_init(struct intel_uncore_type **types, bool setid)
{ … }

/*
 * Get the die information of a PCI device.
 * @pdev: The PCI device.
 * @die: The die id which the device maps to.
 */
static int uncore_pci_get_dev_die_info(struct pci_dev *pdev, int *die)
{ … }

static struct intel_uncore_pmu *
uncore_pci_find_dev_pmu_from_types(struct pci_dev *pdev)
{ … }

/*
 * Find the PMU of a PCI device.
 * @pdev: The PCI device.
 * @ids: The ID table of the available PCI devices with a PMU.
 *       If NULL, search the whole uncore_pci_uncores.
 */
static struct intel_uncore_pmu *
uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids)
{ … }

/*
 * Register the PMU for a PCI device
 * @pdev: The PCI device.
 * @type: The corresponding PMU type of the device.
 * @pmu: The corresponding PMU of the device.
 * @die: The die id which the device maps to.
 */
static int uncore_pci_pmu_register(struct pci_dev *pdev,
				   struct intel_uncore_type *type,
				   struct intel_uncore_pmu *pmu,
				   int die)
{ … }

/*
 * add a pci uncore device
 */
static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{ … }

/*
 * Unregister the PMU of a PCI device
 * @pmu: The corresponding PMU is unregistered.
 * @die: The die id which the device maps to.
 */
static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die)
{ … }

static void uncore_pci_remove(struct pci_dev *pdev)
{ … }

static int uncore_bus_notify(struct notifier_block *nb,
			     unsigned long action, void *data,
			     const struct pci_device_id *ids)
{ … }

static int uncore_pci_sub_bus_notify(struct notifier_block *nb,
				     unsigned long action, void *data)
{ … }

static struct notifier_block uncore_pci_sub_notifier = …;

static void uncore_pci_sub_driver_init(void)
{ … }

static int uncore_pci_bus_notify(struct notifier_block *nb,
				     unsigned long action, void *data)
{ … }

static struct notifier_block uncore_pci_notifier = …;


static void uncore_pci_pmus_register(void)
{ … }

static int __init uncore_pci_init(void)
{ … }

static void uncore_pci_exit(void)
{ … }

static bool uncore_die_has_box(struct intel_uncore_type *type,
			       int die, unsigned int pmu_idx)
{ … }

static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
				   int new_cpu)
{ … }

static void uncore_change_context(struct intel_uncore_type **uncores,
				  int old_cpu, int new_cpu)
{ … }

static void uncore_box_unref(struct intel_uncore_type **types, int id)
{ … }

static int uncore_event_cpu_offline(unsigned int cpu)
{ … }

static int allocate_boxes(struct intel_uncore_type **types,
			 unsigned int die, unsigned int cpu)
{ … }

static int uncore_box_ref(struct intel_uncore_type **types,
			  int id, unsigned int cpu)
{ … }

static int uncore_event_cpu_online(unsigned int cpu)
{ … }

static int __init type_pmu_register(struct intel_uncore_type *type)
{ … }

static int __init uncore_msr_pmus_register(void)
{ … }

static int __init uncore_cpu_init(void)
{ … }

static int __init uncore_mmio_init(void)
{ … }

struct intel_uncore_init_fun { … };

static const struct intel_uncore_init_fun nhm_uncore_init __initconst = …;

static const struct intel_uncore_init_fun snb_uncore_init __initconst = …;

static const struct intel_uncore_init_fun ivb_uncore_init __initconst = …;

static const struct intel_uncore_init_fun hsw_uncore_init __initconst = …;

static const struct intel_uncore_init_fun bdw_uncore_init __initconst = …;

static const struct intel_uncore_init_fun snbep_uncore_init __initconst = …;

static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = …;

static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = …;

static const struct intel_uncore_init_fun hswep_uncore_init __initconst = …;

static const struct intel_uncore_init_fun bdx_uncore_init __initconst = …;

static const struct intel_uncore_init_fun knl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun skl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun skx_uncore_init __initconst = …;

static const struct intel_uncore_init_fun icl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun tgl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = …;

static const struct intel_uncore_init_fun rkl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun adl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun mtl_uncore_init __initconst = …;

static const struct intel_uncore_init_fun icx_uncore_init __initconst = …;

static const struct intel_uncore_init_fun snr_uncore_init __initconst = …;

static const struct intel_uncore_init_fun spr_uncore_init __initconst = …;

static const struct intel_uncore_init_fun gnr_uncore_init __initconst = …;

static const struct intel_uncore_init_fun generic_uncore_init __initconst = …;

static const struct x86_cpu_id intel_uncore_match[] __initconst = …;
MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match);

static int __init intel_uncore_init(void)
{ … }
module_init(…) …;

static void __exit intel_uncore_exit(void)
{ … }
module_exit(intel_uncore_exit);