linux/drivers/gpu/drm/xe/xe_oa.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2023-2024 Intel Corporation
 */

#include <linux/anon_inodes.h>
#include <linux/delay.h>
#include <linux/nospec.h>
#include <linux/poll.h>

#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/xe_drm.h>

#include "abi/guc_actions_slpc_abi.h"
#include "instructions/xe_mi_commands.h"
#include "regs/xe_engine_regs.h"
#include "regs/xe_gt_regs.h"
#include "regs/xe_lrc_layout.h"
#include "regs/xe_oa_regs.h"
#include "xe_assert.h"
#include "xe_bb.h"
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_exec_queue.h"
#include "xe_force_wake.h"
#include "xe_gt.h"
#include "xe_gt_mcr.h"
#include "xe_gt_printk.h"
#include "xe_guc_pc.h"
#include "xe_lrc.h"
#include "xe_macros.h"
#include "xe_mmio.h"
#include "xe_oa.h"
#include "xe_observation.h"
#include "xe_pm.h"
#include "xe_sched_job.h"
#include "xe_sriov.h"

#define DEFAULT_POLL_FREQUENCY_HZ …
#define DEFAULT_POLL_PERIOD_NS …
#define XE_OA_UNIT_INVALID …

struct xe_oa_reg { … };

struct xe_oa_config { … };

struct flex { … };

struct xe_oa_open_param { … };

struct xe_oa_config_bo { … };

#define DRM_FMT(x) …

static const struct xe_oa_format oa_formats[] = …;

static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head)
{ … }

static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n)
{ … }

static void xe_oa_config_release(struct kref *ref)
{ … }

static void xe_oa_config_put(struct xe_oa_config *oa_config)
{ … }

static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config)
{ … }

static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set)
{ … }

static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo)
{ … }

static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream)
{ … }

static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream)
{ … }

#define oa_report_header_64bit(__s) …

static u64 oa_report_id(struct xe_oa_stream *stream, void *report)
{ … }

static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report)
{ … }

static u64 oa_timestamp(struct xe_oa_stream *stream, void *report)
{ … }

static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report)
{ … }

static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)
{ … }

static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer)
{ … }

static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf,
			       size_t count, size_t *offset, const u8 *report)
{ … }

static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,
				size_t count, size_t *offset)
{ … }

static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream)
{ … }

static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask)
{ … }

static u32 __oa_ccs_select(struct xe_oa_stream *stream)
{ … }

static void xe_oa_enable(struct xe_oa_stream *stream)
{ … }

static void xe_oa_disable(struct xe_oa_stream *stream)
{ … }

static int xe_oa_wait_unlocked(struct xe_oa_stream *stream)
{ … }

#define OASTATUS_RELEVANT_BITS …

static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf,
			size_t count, size_t *offset)
{ … }

static ssize_t xe_oa_read(struct file *file, char __user *buf,
			  size_t count, loff_t *ppos)
{ … }

static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream,
				  struct file *file, poll_table *wait)
{ … }

static __poll_t xe_oa_poll(struct file *file, poll_table *wait)
{ … }

static int xe_oa_submit_bb(struct xe_oa_stream *stream, struct xe_bb *bb)
{ … }

static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs)
{ … }

static int num_lri_dwords(int num_regs)
{ … }

static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream)
{ … }

static void xe_oa_free_configs(struct xe_oa_stream *stream)
{ … }

static void xe_oa_store_flex(struct xe_oa_stream *stream, struct xe_lrc *lrc,
			     struct xe_bb *bb, const struct flex *flex, u32 count)
{ … }

static int xe_oa_modify_ctx_image(struct xe_oa_stream *stream, struct xe_lrc *lrc,
				  const struct flex *flex, u32 count)
{ … }

static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri)
{ … }

static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable)
{ … }

static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable)
{ … }

static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable)
{ … }

#define HAS_OA_BPC_REPORTING(xe) …

static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable)
{ … }

static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)
{ … }

static void xe_oa_stream_destroy(struct xe_oa_stream *stream)
{ … }

static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream)
{ … }

static struct xe_oa_config_bo *
__xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
{ … }

static struct xe_oa_config_bo *
xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
{ … }

static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config)
{ … }

static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream)
{ … }

static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
{ … }

static void xe_oa_stream_enable(struct xe_oa_stream *stream)
{ … }

static void xe_oa_stream_disable(struct xe_oa_stream *stream)
{ … }

static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)
{ … }

static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream)
{ … }

static int xe_oa_enable_locked(struct xe_oa_stream *stream)
{ … }

static int xe_oa_disable_locked(struct xe_oa_stream *stream)
{ … }

static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg)
{ … }

static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg)
{ … }

static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg)
{ … }

static long xe_oa_ioctl_locked(struct xe_oa_stream *stream,
			       unsigned int cmd,
			       unsigned long arg)
{ … }

static long xe_oa_ioctl(struct file *file,
			unsigned int cmd,
			unsigned long arg)
{ … }

static void xe_oa_destroy_locked(struct xe_oa_stream *stream)
{ … }

static int xe_oa_release(struct inode *inode, struct file *file)
{ … }

static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
{ … }

static const struct file_operations xe_oa_fops = …;

static bool engine_supports_mi_query(struct xe_hw_engine *hwe)
{ … }

static bool xe_oa_find_reg_in_lri(u32 *state, u32 reg, u32 *offset, u32 end)
{ … }

#define IS_MI_LRI_CMD(x) …

static u32 xe_oa_context_image_offset(struct xe_oa_stream *stream, u32 reg)
{ … }

static int xe_oa_set_ctx_ctrl_offset(struct xe_oa_stream *stream)
{ … }

static int xe_oa_stream_init(struct xe_oa_stream *stream,
			     struct xe_oa_open_param *param)
{ … }

static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa,
					  struct xe_oa_open_param *param)
{ … }

/**
 * xe_oa_timestamp_frequency - Return OA timestamp frequency
 * @gt: @xe_gt
 *
 * OA timestamp frequency = CS timestamp frequency in most platforms. On some
 * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such
 * cases, return the adjusted CS timestamp frequency to the user.
 */
u32 xe_oa_timestamp_frequency(struct xe_gt *gt)
{ … }

static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent)
{ … }

static bool engine_supports_oa_format(const struct xe_hw_engine *hwe, int type)
{ … }

static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name)
{ … }

/**
 * xe_oa_unit_id - Return OA unit ID for a hardware engine
 * @hwe: @xe_hw_engine
 *
 * Return OA unit ID for a hardware engine when available
 */
u16 xe_oa_unit_id(struct xe_hw_engine *hwe)
{ … }

static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value,
				     struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value,
				    struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value,
				     struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value,
				    struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value,
				      struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value,
				   struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value,
					struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value,
					  struct xe_oa_open_param *param)
{ … }

static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value,
				struct xe_oa_open_param *param)
{ … }

xe_oa_set_property_fn;
static const xe_oa_set_property_fn xe_oa_set_property_funcs[] = …;

static int xe_oa_user_ext_set_property(struct xe_oa *oa, u64 extension,
				       struct xe_oa_open_param *param)
{ … }

xe_oa_user_extension_fn;
static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = …;

#define MAX_USER_EXTENSIONS …
static int xe_oa_user_extensions(struct xe_oa *oa, u64 extension, int ext_number,
				 struct xe_oa_open_param *param)
{ … }

/**
 * xe_oa_stream_open_ioctl - Opens an OA stream
 * @dev: @drm_device
 * @data: pointer to struct @drm_xe_oa_config
 * @file: @drm_file
 *
 * The functions opens an OA stream. An OA stream, opened with specified
 * properties, enables OA counter samples to be collected, either
 * periodically (time based sampling), or on request (using OA queries)
 */
int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
{ … }

static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr)
{ … }

static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table)
{ … }

static const struct xe_mmio_range xehp_oa_b_counters[] = …;

static const struct xe_mmio_range gen12_oa_b_counters[] = …;

static const struct xe_mmio_range mtl_oam_b_counters[] = …;

static const struct xe_mmio_range xe2_oa_b_counters[] = …;

static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr)
{ … }

static const struct xe_mmio_range mtl_oa_mux_regs[] = …;

static const struct xe_mmio_range gen12_oa_mux_regs[] = …;

static const struct xe_mmio_range xe2_oa_mux_regs[] = …;

static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr)
{ … }

static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr)
{ … }

static struct xe_oa_reg *
xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr),
		 u32 __user *regs, u32 n_regs)
{ … }

static ssize_t show_dynamic_id(struct kobject *kobj,
			       struct kobj_attribute *attr,
			       char *buf)
{ … }

static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa,
					 struct xe_oa_config *oa_config)
{ … }

/**
 * xe_oa_add_config_ioctl - Adds one OA config
 * @dev: @drm_device
 * @data: pointer to struct @drm_xe_oa_config
 * @file: @drm_file
 *
 * The functions adds an OA config to the set of OA configs maintained in
 * the kernel. The config determines which OA metrics are collected for an
 * OA stream.
 */
int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
{ … }

/**
 * xe_oa_remove_config_ioctl - Removes one OA config
 * @dev: @drm_device
 * @data: pointer to struct @drm_xe_observation_param
 * @file: @drm_file
 */
int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
{ … }

/**
 * xe_oa_register - Xe OA registration
 * @xe: @xe_device
 *
 * Exposes the metrics sysfs directory upon completion of module initialization
 */
void xe_oa_register(struct xe_device *xe)
{ … }

/**
 * xe_oa_unregister - Xe OA de-registration
 * @xe: @xe_device
 */
void xe_oa_unregister(struct xe_device *xe)
{ … }

static u32 num_oa_units_per_gt(struct xe_gt *gt)
{ … }

static u32 __hwe_oam_unit(struct xe_hw_engine *hwe)
{ … }

static u32 __hwe_oa_unit(struct xe_hw_engine *hwe)
{ … }

static struct xe_oa_regs __oam_regs(u32 base)
{ … }

static struct xe_oa_regs __oag_regs(void)
{ … }

static void __xe_oa_init_oa_units(struct xe_gt *gt)
{ … }

static int xe_oa_init_gt(struct xe_gt *gt)
{ … }

static int xe_oa_init_oa_units(struct xe_oa *oa)
{ … }

static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format)
{ … }

static void xe_oa_init_supported_formats(struct xe_oa *oa)
{ … }

/**
 * xe_oa_init - OA initialization during device probe
 * @xe: @xe_device
 *
 * Return: 0 on success or a negative error code on failure
 */
int xe_oa_init(struct xe_device *xe)
{ … }

static int destroy_config(int id, void *p, void *data)
{ … }

/**
 * xe_oa_fini - OA de-initialization during device remove
 * @xe: @xe_device
 */
void xe_oa_fini(struct xe_device *xe)
{ … }