#include <linux/delay.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include "hwmgr.h"
#include "amd_powerplay.h"
#include "hardwaremanager.h"
#include "ppatomfwctrl.h"
#include "atomfirmware.h"
#include "cgs_common.h"
#include "vega10_powertune.h"
#include "smu9.h"
#include "smu9_driver_if.h"
#include "vega10_inc.h"
#include "soc15_common.h"
#include "pppcielanes.h"
#include "vega10_hwmgr.h"
#include "vega10_smumgr.h"
#include "vega10_processpptables.h"
#include "vega10_pptable.h"
#include "vega10_thermal.h"
#include "pp_debug.h"
#include "amd_pcie_helpers.h"
#include "ppinterrupt.h"
#include "pp_overdriver.h"
#include "pp_thermal.h"
#include "vega10_baco.h"
#include "smuio/smuio_9_0_offset.h"
#include "smuio/smuio_9_0_sh_mask.h"
#define smnPCIE_LC_SPEED_CNTL …
#define smnPCIE_LC_LINK_WIDTH_CNTL …
#define HBM_MEMORY_CHANNEL_WIDTH …
static const uint32_t channel_number[] = …;
#define mmDF_CS_AON0_DramBaseAddress0 …
#define mmDF_CS_AON0_DramBaseAddress0_BASE_IDX …
#define DF_CS_AON0_DramBaseAddress0__AddrRngVal__SHIFT …
#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn__SHIFT …
#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT …
#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel__SHIFT …
#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr__SHIFT …
#define DF_CS_AON0_DramBaseAddress0__AddrRngVal_MASK …
#define DF_CS_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK …
#define DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK …
#define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel_MASK …
#define DF_CS_AON0_DramBaseAddress0__DramBaseAddr_MASK …
CLOCK_ID_e;
static const ULONG PhwVega10_Magic = …;
static struct vega10_power_state *cast_phw_vega10_power_state(
struct pp_hw_power_state *hw_ps)
{ … }
static const struct vega10_power_state *cast_const_phw_vega10_power_state(
const struct pp_hw_power_state *hw_ps)
{ … }
static void vega10_set_default_registry_data(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_set_features_platform_caps(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_odn_initial_default_setting(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
{ … }
#ifdef PPLIB_VEGA10_EVV_SUPPORT
static int vega10_get_socclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
phm_ppt_v1_voltage_lookup_table *lookup_table,
uint16_t virtual_voltage_id, int32_t *socclk)
{
uint8_t entry_id;
uint8_t voltage_id;
struct phm_ppt_v2_information *table_info =
(struct phm_ppt_v2_information *)(hwmgr->pptable);
PP_ASSERT_WITH_CODE(lookup_table->count != 0,
"Lookup table is empty",
return -EINVAL);
for (entry_id = 0; entry_id < table_info->vdd_dep_on_sclk->count; entry_id++) {
voltage_id = table_info->vdd_dep_on_socclk->entries[entry_id].vddInd;
if (lookup_table->entries[voltage_id].us_vdd == virtual_voltage_id)
break;
}
PP_ASSERT_WITH_CODE(entry_id < table_info->vdd_dep_on_socclk->count,
"Can't find requested voltage id in vdd_dep_on_socclk table!",
return -EINVAL);
*socclk = table_info->vdd_dep_on_socclk->entries[entry_id].clk;
return 0;
}
#define ATOM_VIRTUAL_VOLTAGE_ID0 …
static int vega10_get_evv_voltages(struct pp_hwmgr *hwmgr)
{
struct vega10_hwmgr *data = hwmgr->backend;
uint16_t vv_id;
uint32_t vddc = 0;
uint16_t i, j;
uint32_t sclk = 0;
struct phm_ppt_v2_information *table_info =
(struct phm_ppt_v2_information *)hwmgr->pptable;
struct phm_ppt_v1_clock_voltage_dependency_table *socclk_table =
table_info->vdd_dep_on_socclk;
int result;
for (i = 0; i < VEGA10_MAX_LEAKAGE_COUNT; i++) {
vv_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
if (!vega10_get_socclk_for_voltage_evv(hwmgr,
table_info->vddc_lookup_table, vv_id, &sclk)) {
if (PP_CAP(PHM_PlatformCaps_ClockStretcher)) {
for (j = 1; j < socclk_table->count; j++) {
if (socclk_table->entries[j].clk == sclk &&
socclk_table->entries[j].cks_enable == 0) {
sclk += 5000;
break;
}
}
}
PP_ASSERT_WITH_CODE(!atomctrl_get_voltage_evv_on_sclk_ai(hwmgr,
VOLTAGE_TYPE_VDDC, sclk, vv_id, &vddc),
"Error retrieving EVV voltage value!",
continue);
PP_ASSERT_WITH_CODE((vddc < 2000 && vddc != 0),
"Invalid VDDC value", result = -EINVAL;);
if (vddc != 0 && vddc != vv_id) {
data->vddc_leakage.actual_voltage[data->vddc_leakage.count] = (uint16_t)(vddc/100);
data->vddc_leakage.leakage_id[data->vddc_leakage.count] = vv_id;
data->vddc_leakage.count++;
}
}
}
return 0;
}
static void vega10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
uint16_t *voltage, struct vega10_leakage_voltage *leakage_table)
{
uint32_t index;
for (index = 0; index < leakage_table->count; index++) {
if (leakage_table->leakage_id[index] == *voltage) {
*voltage = leakage_table->actual_voltage[index];
break;
}
}
if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
pr_info("Voltage value looks like a Leakage ID but it's not patched\n");
}
static int vega10_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr,
phm_ppt_v1_voltage_lookup_table *lookup_table,
struct vega10_leakage_voltage *leakage_table)
{
uint32_t i;
for (i = 0; i < lookup_table->count; i++)
vega10_patch_with_vdd_leakage(hwmgr,
&lookup_table->entries[i].us_vdd, leakage_table);
return 0;
}
static int vega10_patch_clock_voltage_limits_with_vddc_leakage(
struct pp_hwmgr *hwmgr, struct vega10_leakage_voltage *leakage_table,
uint16_t *vddc)
{
vega10_patch_with_vdd_leakage(hwmgr, (uint16_t *)vddc, leakage_table);
return 0;
}
#endif
static int vega10_patch_voltage_dependency_tables_with_lookup_table(
struct pp_hwmgr *hwmgr)
{ … }
static int vega10_sort_lookup_table(struct pp_hwmgr *hwmgr,
struct phm_ppt_v1_voltage_lookup_table *lookup_table)
{ … }
static int vega10_complete_dependency_tables(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_set_private_data_based_on_pptable(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_init_sclk_threshold(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_setup_dpm_led_config(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_setup_asic_task(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_trim_voltage_table(struct pp_hwmgr *hwmgr,
struct pp_atomfwctrl_voltage_table *vol_table)
{ … }
static int vega10_get_mvdd_voltage_table(struct pp_hwmgr *hwmgr,
phm_ppt_v1_clock_voltage_dependency_table *dep_table,
struct pp_atomfwctrl_voltage_table *vol_table)
{ … }
static int vega10_get_vddci_voltage_table(struct pp_hwmgr *hwmgr,
phm_ppt_v1_clock_voltage_dependency_table *dep_table,
struct pp_atomfwctrl_voltage_table *vol_table)
{ … }
static int vega10_get_vdd_voltage_table(struct pp_hwmgr *hwmgr,
phm_ppt_v1_clock_voltage_dependency_table *dep_table,
struct pp_atomfwctrl_voltage_table *vol_table)
{ … }
static void vega10_trim_voltage_table_to_fit_state_table(
struct pp_hwmgr *hwmgr,
uint32_t max_vol_steps,
struct pp_atomfwctrl_voltage_table *vol_table)
{ … }
static int vega10_construct_voltage_tables(struct pp_hwmgr *hwmgr)
{ … }
static void vega10_init_dpm_state(struct vega10_dpm_state *dpm_state)
{ … }
static void vega10_setup_default_single_dpm_table(struct pp_hwmgr *hwmgr,
struct vega10_single_dpm_table *dpm_table,
struct phm_ppt_v1_clock_voltage_dependency_table *dep_table)
{ … }
static int vega10_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_ulv_state(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_single_lclk_level(struct pp_hwmgr *hwmgr,
uint32_t lclock, uint8_t *curr_lclk_did)
{ … }
static int vega10_override_pcie_parameters(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_smc_link_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr,
uint32_t gfx_clock, PllSetting_t *current_gfxclk_level,
uint32_t *acg_freq)
{ … }
static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr,
uint32_t soc_clock, uint8_t *current_soc_did,
uint8_t *current_vol_index)
{ … }
static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
{ … }
static void vega10_populate_vddc_soc_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_single_memory_level(struct pp_hwmgr *hwmgr,
uint32_t mem_clock, uint8_t *current_mem_vid,
PllSetting_t *current_memclk_level, uint8_t *current_mem_soc_vind)
{ … }
static int vega10_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_single_display_type(struct pp_hwmgr *hwmgr,
DSPCLK_e disp_clock)
{ … }
static int vega10_populate_all_display_clock_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_single_eclock_level(struct pp_hwmgr *hwmgr,
uint32_t eclock, uint8_t *current_eclk_did,
uint8_t *current_soc_vol)
{ … }
static int vega10_populate_smc_vce_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_single_vclock_level(struct pp_hwmgr *hwmgr,
uint32_t vclock, uint8_t *current_vclk_did)
{ … }
static int vega10_populate_single_dclock_level(struct pp_hwmgr *hwmgr,
uint32_t dclock, uint8_t *current_dclk_did)
{ … }
static int vega10_populate_smc_uvd_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_clock_stretcher_table(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_acg_enable(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_acg_disable(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_gpio_parameters(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_avfs_enable(struct pp_hwmgr *hwmgr, bool enable)
{ … }
static int vega10_update_avfs(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_populate_and_upload_avfs_fuse_override(struct pp_hwmgr *hwmgr)
{ … }
static void vega10_check_dpm_table_updated(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_init_smc_table(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_enable_thermal_protection(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_disable_thermal_protection(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_enable_vrhot_feature(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_enable_ulv(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_disable_ulv(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_stop_dpm(struct pp_hwmgr *hwmgr, uint32_t bitmap)
{ … }
static int vega10_start_dpm(struct pp_hwmgr *hwmgr, uint32_t bitmap)
{ … }
static int vega10_enable_disable_PCC_limit_feature(struct pp_hwmgr *hwmgr, bool enable)
{ … }
static void vega10_populate_umdpstate_clocks(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_power_state_size(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr,
void *state, struct pp_power_state *power_state,
void *pp_table, uint32_t classification_flag)
{ … }
static int vega10_get_pp_table_entry(struct pp_hwmgr *hwmgr,
unsigned long entry_index, struct pp_power_state *state)
{ … }
static int vega10_patch_boot_state(struct pp_hwmgr *hwmgr,
struct pp_hw_power_state *hw_ps)
{ … }
static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
struct pp_power_state *request_ps,
const struct pp_power_state *current_ps)
{ … }
static int vega10_find_dpm_states_clocks_in_dpm_table(struct pp_hwmgr *hwmgr, const void *input)
{ … }
static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
struct pp_hwmgr *hwmgr, const void *input)
{ … }
static int vega10_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
struct vega10_single_dpm_table *dpm_table,
uint32_t low_limit, uint32_t high_limit)
{ … }
static int vega10_trim_single_dpm_states_with_mask(struct pp_hwmgr *hwmgr,
struct vega10_single_dpm_table *dpm_table,
uint32_t low_limit, uint32_t high_limit,
uint32_t disable_dpm_mask)
{ … }
static int vega10_trim_dpm_states(struct pp_hwmgr *hwmgr,
const struct vega10_power_state *vega10_ps)
{ … }
static uint32_t vega10_find_lowest_dpm_level(
struct vega10_single_dpm_table *table)
{ … }
static uint32_t vega10_find_highest_dpm_level(
struct vega10_single_dpm_table *table)
{ … }
static void vega10_apply_dal_minimum_voltage_request(
struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_soc_index_for_max_uclk(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_upload_dpm_bootup_level(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_upload_dpm_max_level(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_generate_dpm_level_enable_mask(
struct pp_hwmgr *hwmgr, const void *input)
{ … }
int vega10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable)
{ … }
static int vega10_update_sclk_threshold(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_set_power_state_tasks(struct pp_hwmgr *hwmgr,
const void *input)
{ … }
static uint32_t vega10_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
{ … }
static uint32_t vega10_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
{ … }
static int vega10_get_gpu_power(struct pp_hwmgr *hwmgr,
uint32_t *query)
{ … }
static int vega10_read_sensor(struct pp_hwmgr *hwmgr, int idx,
void *value, int *size)
{ … }
static void vega10_notify_smc_display_change(struct pp_hwmgr *hwmgr,
bool has_disp)
{ … }
static int vega10_display_clock_voltage_request(struct pp_hwmgr *hwmgr,
struct pp_display_clock_request *clock_req)
{ … }
static uint8_t vega10_get_uclk_index(struct pp_hwmgr *hwmgr,
struct phm_ppt_v1_clock_voltage_dependency_table *mclk_table,
uint32_t frequency)
{ … }
static int vega10_notify_smc_display_config_after_ps_adjustment(
struct pp_hwmgr *hwmgr)
{ … }
static int vega10_force_dpm_highest(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_force_dpm_lowest(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_profiling_clk_mask(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level,
uint32_t *sclk_mask, uint32_t *mclk_mask, uint32_t *soc_mask)
{ … }
static void vega10_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
{ … }
static int vega10_force_clock_level(struct pp_hwmgr *hwmgr,
enum pp_clock_type type, uint32_t mask)
{ … }
static int vega10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
enum amd_dpm_forced_level level)
{ … }
static uint32_t vega10_get_fan_control_mode(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_dal_power_level(struct pp_hwmgr *hwmgr,
struct amd_pp_simple_clock_info *info)
{ … }
static void vega10_get_sclks(struct pp_hwmgr *hwmgr,
struct pp_clock_levels_with_latency *clocks)
{ … }
static void vega10_get_memclocks(struct pp_hwmgr *hwmgr,
struct pp_clock_levels_with_latency *clocks)
{ … }
static void vega10_get_dcefclocks(struct pp_hwmgr *hwmgr,
struct pp_clock_levels_with_latency *clocks)
{ … }
static void vega10_get_socclocks(struct pp_hwmgr *hwmgr,
struct pp_clock_levels_with_latency *clocks)
{ … }
static int vega10_get_clock_by_type_with_latency(struct pp_hwmgr *hwmgr,
enum amd_pp_clock_type type,
struct pp_clock_levels_with_latency *clocks)
{ … }
static int vega10_get_clock_by_type_with_voltage(struct pp_hwmgr *hwmgr,
enum amd_pp_clock_type type,
struct pp_clock_levels_with_voltage *clocks)
{ … }
static int vega10_set_watermarks_for_clocks_ranges(struct pp_hwmgr *hwmgr,
void *clock_range)
{ … }
static int vega10_get_ppfeature_status(struct pp_hwmgr *hwmgr, char *buf)
{ … }
static int vega10_set_ppfeature_status(struct pp_hwmgr *hwmgr, uint64_t new_ppfeature_masks)
{ … }
static int vega10_get_current_pcie_link_width_level(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_current_pcie_link_speed_level(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_emit_clock_levels(struct pp_hwmgr *hwmgr,
enum pp_clock_type type, char *buf, int *offset)
{ … }
static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
enum pp_clock_type type, char *buf)
{ … }
static int vega10_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_enable_disable_uvd_dpm(struct pp_hwmgr *hwmgr, bool enable)
{ … }
static void vega10_power_gate_vce(struct pp_hwmgr *hwmgr, bool bgate)
{ … }
static void vega10_power_gate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
{ … }
static inline bool vega10_are_power_levels_equal(
const struct vega10_performance_level *pl1,
const struct vega10_performance_level *pl2)
{ … }
static int vega10_check_states_equal(struct pp_hwmgr *hwmgr,
const struct pp_hw_power_state *pstate1,
const struct pp_hw_power_state *pstate2, bool *equal)
{ … }
static bool
vega10_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_power_off_asic(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_get_sclk_od(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
{ … }
static int vega10_get_mclk_od(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
{ … }
static int vega10_notify_cac_buffer_info(struct pp_hwmgr *hwmgr,
uint32_t virtual_addr_low,
uint32_t virtual_addr_hi,
uint32_t mc_addr_low,
uint32_t mc_addr_hi,
uint32_t size)
{ … }
static int vega10_get_thermal_temperature_range(struct pp_hwmgr *hwmgr,
struct PP_TemperatureRange *thermal_data)
{ … }
static int vega10_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf)
{ … }
static bool vega10_get_power_profile_mode_quirks(struct pp_hwmgr *hwmgr)
{ … }
static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, uint32_t size)
{ … }
static bool vega10_check_clk_voltage_valid(struct pp_hwmgr *hwmgr,
enum PP_OD_DPM_TABLE_COMMAND type,
uint32_t clk,
uint32_t voltage)
{ … }
static void vega10_odn_update_power_state(struct pp_hwmgr *hwmgr)
{ … }
static void vega10_odn_update_soc_table(struct pp_hwmgr *hwmgr,
enum PP_OD_DPM_TABLE_COMMAND type)
{ … }
static int vega10_odn_edit_dpm_table(struct pp_hwmgr *hwmgr,
enum PP_OD_DPM_TABLE_COMMAND type,
long *input, uint32_t size)
{ … }
static int vega10_set_mp1_state(struct pp_hwmgr *hwmgr,
enum pp_mp1_state mp1_state)
{ … }
static int vega10_get_performance_level(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *state,
PHM_PerformanceLevelDesignation designation, uint32_t index,
PHM_PerformanceLevel *level)
{ … }
static int vega10_disable_power_features_for_compute_performance(struct pp_hwmgr *hwmgr, bool disable)
{ … }
static const struct pp_hwmgr_func vega10_hwmgr_funcs = …;
int vega10_hwmgr_init(struct pp_hwmgr *hwmgr)
{ … }