intel_rps.c | Explore in Territory

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2019 Intel Corporation
 */

#include <linux/string_helpers.h>

#include <drm/intel/i915_drm.h>

#include "display/intel_display.h"
#include "display/intel_display_irq.h"
#include "i915_drv.h"
#include "i915_irq.h"
#include "i915_reg.h"
#include "intel_breadcrumbs.h"
#include "intel_gt.h"
#include "intel_gt_clock_utils.h"
#include "intel_gt_irq.h"
#include "intel_gt_pm.h"
#include "intel_gt_pm_irq.h"
#include "intel_gt_print.h"
#include "intel_gt_regs.h"
#include "intel_mchbar_regs.h"
#include "intel_pcode.h"
#include "intel_rps.h"
#include "vlv_sideband.h"
#include "../../../platform/x86/intel_ips.h"

#define BUSY_MAX_EI …

/*
 * Lock protecting IPS related data structures
 */
static DEFINE_SPINLOCK(mchdev_lock);

static struct intel_gt *rps_to_gt(struct intel_rps *rps)
{ … }

static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
{ … }

static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
{ … }

static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
{ … }

static bool rps_uses_slpc(struct intel_rps *rps)
{ … }

static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
{ … }

static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
{ … }

static void rps_timer(struct timer_list *t)
{ … }

static void rps_start_timer(struct intel_rps *rps)
{ … }

static void rps_stop_timer(struct intel_rps *rps)
{ … }

static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
{ … }

static void rps_reset_ei(struct intel_rps *rps)
{ … }

static void rps_enable_interrupts(struct intel_rps *rps)
{ … }

static void gen6_rps_reset_interrupts(struct intel_rps *rps)
{ … }

static void gen11_rps_reset_interrupts(struct intel_rps *rps)
{ … }

static void rps_reset_interrupts(struct intel_rps *rps)
{ … }

static void rps_disable_interrupts(struct intel_rps *rps)
{ … }

static const struct cparams { … } cparams[] = …;

static void gen5_rps_init(struct intel_rps *rps)
{ … }

static unsigned long
__ips_chipset_val(struct intel_ips *ips)
{ … }

static unsigned long ips_mch_val(struct intel_uncore *uncore)
{ … }

static int _pxvid_to_vd(u8 pxvid)
{ … }

static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
{ … }

static void __gen5_ips_update(struct intel_ips *ips)
{ … }

static void gen5_rps_update(struct intel_rps *rps)
{ … }

static unsigned int gen5_invert_freq(struct intel_rps *rps,
				     unsigned int val)
{ … }

static int __gen5_rps_set(struct intel_rps *rps, u8 val)
{ … }

static int gen5_rps_set(struct intel_rps *rps, u8 val)
{ … }

static unsigned long intel_pxfreq(u32 vidfreq)
{ … }

static unsigned int init_emon(struct intel_uncore *uncore)
{ … }

static bool gen5_rps_enable(struct intel_rps *rps)
{ … }

static void gen5_rps_disable(struct intel_rps *rps)
{ … }

static u32 rps_limits(struct intel_rps *rps, u8 val)
{ … }

static void rps_set_power(struct intel_rps *rps, int new_power)
{ … }

static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
{ … }

void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
{ … }

static int gen6_rps_set(struct intel_rps *rps, u8 val)
{ … }

static int vlv_rps_set(struct intel_rps *rps, u8 val)
{ … }

static int rps_set(struct intel_rps *rps, u8 val, bool update)
{ … }

void intel_rps_unpark(struct intel_rps *rps)
{ … }

void intel_rps_park(struct intel_rps *rps)
{ … }

u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
{ … }

static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
{ … }

int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
{ … }

void intel_rps_dec_waiters(struct intel_rps *rps)
{ … }

void intel_rps_boost(struct i915_request *rq)
{ … }

int intel_rps_set(struct intel_rps *rps, u8 val)
{ … }

static u32 intel_rps_read_state_cap(struct intel_rps *rps)
{ … }

static void
mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
{ … }

static void
__gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
{ … }

/**
 * gen6_rps_get_freq_caps - Get freq caps exposed by HW
 * @rps: the intel_rps structure
 * @caps: returned freq caps
 *
 * Returned "caps" frequencies should be converted to MHz using
 * intel_gpu_freq()
 */
void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
{ … }

static void gen6_rps_init(struct intel_rps *rps)
{ … }

static bool rps_reset(struct intel_rps *rps)
{ … }

/* See the Gen9_GT_PM_Programming_Guide doc for the below */
static bool gen9_rps_enable(struct intel_rps *rps)
{ … }

static bool gen8_rps_enable(struct intel_rps *rps)
{ … }

static bool gen6_rps_enable(struct intel_rps *rps)
{ … }

static int chv_rps_max_freq(struct intel_rps *rps)
{ … }

static int chv_rps_rpe_freq(struct intel_rps *rps)
{ … }

static int chv_rps_guar_freq(struct intel_rps *rps)
{ … }

static u32 chv_rps_min_freq(struct intel_rps *rps)
{ … }

static bool chv_rps_enable(struct intel_rps *rps)
{ … }

static int vlv_rps_guar_freq(struct intel_rps *rps)
{ … }

static int vlv_rps_max_freq(struct intel_rps *rps)
{ … }

static int vlv_rps_rpe_freq(struct intel_rps *rps)
{ … }

static int vlv_rps_min_freq(struct intel_rps *rps)
{ … }

static bool vlv_rps_enable(struct intel_rps *rps)
{ … }

static unsigned long __ips_gfx_val(struct intel_ips *ips)
{ … }

static bool has_busy_stats(struct intel_rps *rps)
{ … }

void intel_rps_enable(struct intel_rps *rps)
{ … }

static void gen6_rps_disable(struct intel_rps *rps)
{ … }

void intel_rps_disable(struct intel_rps *rps)
{ … }

static int byt_gpu_freq(struct intel_rps *rps, int val)
{ … }

static int byt_freq_opcode(struct intel_rps *rps, int val)
{ … }

static int chv_gpu_freq(struct intel_rps *rps, int val)
{ … }

static int chv_freq_opcode(struct intel_rps *rps, int val)
{ … }

int intel_gpu_freq(struct intel_rps *rps, int val)
{ … }

int intel_freq_opcode(struct intel_rps *rps, int val)
{ … }

static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
{ … }

static void vlv_rps_init(struct intel_rps *rps)
{ … }

static void chv_rps_init(struct intel_rps *rps)
{ … }

static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
{ … }

static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
{ … }

static void rps_work(struct work_struct *work)
{ … }

void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
{ … }

void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
{ … }

void gen5_rps_irq_handler(struct intel_rps *rps)
{ … }

void intel_rps_init_early(struct intel_rps *rps)
{ … }

void intel_rps_init(struct intel_rps *rps)
{ … }

void intel_rps_sanitize(struct intel_rps *rps)
{ … }

u32 intel_rps_read_rpstat(struct intel_rps *rps)
{ … }

static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
{ … }

static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
{ … }

static u32 read_cagf(struct intel_rps *rps)
{ … }

u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
{ … }

u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
{ … }

static u32 intel_rps_read_punit_req(struct intel_rps *rps)
{ … }

static u32 intel_rps_get_req(u32 pureq)
{ … }

u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
{ … }

u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
{ … }

u32 intel_rps_get_max_frequency(struct intel_rps *rps)
{ … }

/**
 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
 * @rps: the intel_rps structure
 *
 * Returns the max frequency in a raw format. In newer platforms raw is in
 * units of 50 MHz.
 */
u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
{ … }

u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
{ … }

u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
{ … }

u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
{ … }

static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
{ … }

static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
{ … }

void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
{ … }

static int set_max_freq(struct intel_rps *rps, u32 val)
{ … }

int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
{ … }

u32 intel_rps_get_min_frequency(struct intel_rps *rps)
{ … }

/**
 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
 * @rps: the intel_rps structure
 *
 * Returns the min frequency in a raw format. In newer platforms raw is in
 * units of 50 MHz.
 */
u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
{ … }

static int set_min_freq(struct intel_rps *rps, u32 val)
{ … }

int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
{ … }

u8 intel_rps_get_up_threshold(struct intel_rps *rps)
{ … }

static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
{ … }

int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
{ … }

u8 intel_rps_get_down_threshold(struct intel_rps *rps)
{ … }

int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
{ … }

static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
{ … }

void intel_rps_raise_unslice(struct intel_rps *rps)
{ … }

void intel_rps_lower_unslice(struct intel_rps *rps)
{ … }

static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
{ … }

bool rps_read_mask_mmio(struct intel_rps *rps,
			i915_reg_t reg32, u32 mask)
{ … }

/* External interface for intel_ips.ko */

static struct drm_i915_private __rcu *ips_mchdev;

/*
 * Tells the intel_ips driver that the i915 driver is now loaded, if
 * IPS got loaded first.
 *
 * This awkward dance is so that neither module has to depend on the
 * other in order for IPS to do the appropriate communication of
 * GPU turbo limits to i915.
 */
static void
ips_ping_for_i915_load(void)
{ … }

void intel_rps_driver_register(struct intel_rps *rps)
{ … }

void intel_rps_driver_unregister(struct intel_rps *rps)
{ … }

static struct drm_i915_private *mchdev_get(void)
{ … }

/**
 * i915_read_mch_val - return value for IPS use
 *
 * Calculate and return a value for the IPS driver to use when deciding whether
 * we have thermal and power headroom to increase CPU or GPU power budget.
 */
unsigned long i915_read_mch_val(void)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * i915_gpu_raise - raise GPU frequency limit
 *
 * Raise the limit; IPS indicates we have thermal headroom.
 */
bool i915_gpu_raise(void)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * i915_gpu_lower - lower GPU frequency limit
 *
 * IPS indicates we're close to a thermal limit, so throttle back the GPU
 * frequency maximum.
 */
bool i915_gpu_lower(void)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * i915_gpu_busy - indicate GPU business to IPS
 *
 * Tell the IPS driver whether or not the GPU is busy.
 */
bool i915_gpu_busy(void)
{ … }
EXPORT_SYMBOL_GPL(…);

/**
 * i915_gpu_turbo_disable - disable graphics turbo
 *
 * Disable graphics turbo by resetting the max frequency and setting the
 * current frequency to the default.
 */
bool i915_gpu_turbo_disable(void)
{ … }
EXPORT_SYMBOL_GPL(…);

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftest_rps.c"
#include "selftest_slpc.c"
#endif
linux/drivers/gpu/drm/i915/gt/intel_rps.c
