// SPDX-License-Identifier: MIT /* * Copyright © 2023 Intel Corporation */ #include "i915_drv.h" #include "i915_reg.h" #include "i9xx_wm.h" #include "intel_atomic.h" #include "intel_display.h" #include "intel_display_trace.h" #include "intel_mchbar_regs.h" #include "intel_wm.h" #include "skl_watermark.h" #include "vlv_sideband.h" /* used in computing the new watermarks state */ struct intel_wm_config { … }; struct cxsr_latency { … }; static const struct cxsr_latency cxsr_latency_table[] = …; static const struct cxsr_latency *pnv_get_cxsr_latency(struct drm_i915_private *i915) { … } static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable) { … } static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) { … } #define FW_WM … static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) { … } /** * intel_set_memory_cxsr - Configure CxSR state * @dev_priv: i915 device * @enable: Allow vs. disallow CxSR * * Allow or disallow the system to enter a special CxSR * (C-state self refresh) state. What typically happens in CxSR mode * is that several display FIFOs may get combined into a single larger * FIFO for a particular plane (so called max FIFO mode) to allow the * system to defer memory fetches longer, and the memory will enter * self refresh. * * Note that enabling CxSR does not guarantee that the system enter * this special mode, nor does it guarantee that the system stays * in that mode once entered. So this just allows/disallows the system * to autonomously utilize the CxSR mode. Other factors such as core * C-states will affect when/if the system actually enters/exits the * CxSR mode. * * Note that on VLV/CHV this actually only controls the max FIFO mode, * and the system is free to enter/exit memory self refresh at any time * even when the use of CxSR has been disallowed. * * While the system is actually in the CxSR/max FIFO mode, some plane * control registers will not get latched on vblank. Thus in order to * guarantee the system will respond to changes in the plane registers * we must always disallow CxSR prior to making changes to those registers. * Unfortunately the system will re-evaluate the CxSR conditions at * frame start which happens after vblank start (which is when the plane * registers would get latched), so we can't proceed with the plane update * during the same frame where we disallowed CxSR. * * Certain platforms also have a deeper HPLL SR mode. Fortunately the * HPLL SR mode depends on CxSR itself, so we don't have to hand hold * the hardware w.r.t. HPLL SR when writing to plane registers. * Disallowing just CxSR is sufficient. */ bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) { … } /* * Latency for FIFO fetches is dependent on several factors: * - memory configuration (speed, channels) * - chipset * - current MCH state * It can be fairly high in some situations, so here we assume a fairly * pessimal value. It's a tradeoff between extra memory fetches (if we * set this value too high, the FIFO will fetch frequently to stay full) * and power consumption (set it too low to save power and we might see * FIFO underruns and display "flicker"). * * A value of 5us seems to be a good balance; safe for very low end * platforms but not overly aggressive on lower latency configs. */ static const int pessimal_latency_ns = …; #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) … static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) { … } static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { … } static int i830_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { … } static int i845_get_fifo_size(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { … } /* Pineview has different values for various configs */ static const struct intel_watermark_params pnv_display_wm = …; static const struct intel_watermark_params pnv_display_hplloff_wm = …; static const struct intel_watermark_params pnv_cursor_wm = …; static const struct intel_watermark_params pnv_cursor_hplloff_wm = …; static const struct intel_watermark_params i965_cursor_wm_info = …; static const struct intel_watermark_params i945_wm_info = …; static const struct intel_watermark_params i915_wm_info = …; static const struct intel_watermark_params i830_a_wm_info = …; static const struct intel_watermark_params i830_bc_wm_info = …; static const struct intel_watermark_params i845_wm_info = …; /** * intel_wm_method1 - Method 1 / "small buffer" watermark formula * @pixel_rate: Pipe pixel rate in kHz * @cpp: Plane bytes per pixel * @latency: Memory wakeup latency in 0.1us units * * Compute the watermark using the method 1 or "small buffer" * formula. The caller may additonally add extra cachelines * to account for TLB misses and clock crossings. * * This method is concerned with the short term drain rate * of the FIFO, ie. it does not account for blanking periods * which would effectively reduce the average drain rate across * a longer period. The name "small" refers to the fact the * FIFO is relatively small compared to the amount of data * fetched. * * The FIFO level vs. time graph might look something like: * * |\ |\ * | \ | \ * __---__---__ (- plane active, _ blanking) * -> time * * or perhaps like this: * * |\|\ |\|\ * __----__----__ (- plane active, _ blanking) * -> time * * Returns: * The watermark in bytes */ static unsigned int intel_wm_method1(unsigned int pixel_rate, unsigned int cpp, unsigned int latency) { … } /** * intel_wm_method2 - Method 2 / "large buffer" watermark formula * @pixel_rate: Pipe pixel rate in kHz * @htotal: Pipe horizontal total * @width: Plane width in pixels * @cpp: Plane bytes per pixel * @latency: Memory wakeup latency in 0.1us units * * Compute the watermark using the method 2 or "large buffer" * formula. The caller may additonally add extra cachelines * to account for TLB misses and clock crossings. * * This method is concerned with the long term drain rate * of the FIFO, ie. it does account for blanking periods * which effectively reduce the average drain rate across * a longer period. The name "large" refers to the fact the * FIFO is relatively large compared to the amount of data * fetched. * * The FIFO level vs. time graph might look something like: * * |\___ |\___ * | \___ | \___ * | \ | \ * __ --__--__--__--__--__--__ (- plane active, _ blanking) * -> time * * Returns: * The watermark in bytes */ static unsigned int intel_wm_method2(unsigned int pixel_rate, unsigned int htotal, unsigned int width, unsigned int cpp, unsigned int latency) { … } /** * intel_calculate_wm - calculate watermark level * @i915: the device * @pixel_rate: pixel clock * @wm: chip FIFO params * @fifo_size: size of the FIFO buffer * @cpp: bytes per pixel * @latency_ns: memory latency for the platform * * Calculate the watermark level (the level at which the display plane will * start fetching from memory again). Each chip has a different display * FIFO size and allocation, so the caller needs to figure that out and pass * in the correct intel_watermark_params structure. * * As the pixel clock runs, the FIFO will be drained at a rate that depends * on the pixel size. When it reaches the watermark level, it'll start * fetching FIFO line sized based chunks from memory until the FIFO fills * past the watermark point. If the FIFO drains completely, a FIFO underrun * will occur, and a display engine hang could result. */ static unsigned int intel_calculate_wm(struct drm_i915_private *i915, int pixel_rate, const struct intel_watermark_params *wm, int fifo_size, int cpp, unsigned int latency_ns) { … } static bool is_disabling(int old, int new, int threshold) { … } static bool is_enabling(int old, int new, int threshold) { … } static bool intel_crtc_active(struct intel_crtc *crtc) { … } static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv) { … } static void pnv_update_wm(struct drm_i915_private *dev_priv) { … } /* * Documentation says: * "If the line size is small, the TLB fetches can get in the way of the * data fetches, causing some lag in the pixel data return which is not * accounted for in the above formulas. The following adjustment only * needs to be applied if eight whole lines fit in the buffer at once. * The WM is adjusted upwards by the difference between the FIFO size * and the size of 8 whole lines. This adjustment is always performed * in the actual pixel depth regardless of whether FBC is enabled or not." */ static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp) { … } static void g4x_write_wm_values(struct drm_i915_private *dev_priv, const struct g4x_wm_values *wm) { … } #define FW_WM_VLV … static void vlv_write_wm_values(struct drm_i915_private *dev_priv, const struct vlv_wm_values *wm) { … } #undef FW_WM_VLV static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv) { … } static int g4x_plane_fifo_size(enum plane_id plane_id, int level) { … } static int g4x_fbc_fifo_size(int level) { … } static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, int level) { … } static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state, int level, enum plane_id plane_id, u16 value) { … } static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state, int level, u16 value) { … } static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, u32 pri_val); static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { … } static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, enum plane_id plane_id, int level) { … } static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level) { … } /* mark all levels starting from 'level' as invalid */ static void g4x_invalidate_wms(struct intel_crtc *crtc, struct g4x_wm_state *wm_state, int level) { … } static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state, int level) { … } static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state) { … } static int g4x_compute_pipe_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static int g4x_compute_intermediate_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void g4x_merge_wm(struct drm_i915_private *dev_priv, struct g4x_wm_values *wm) { … } static void g4x_program_watermarks(struct drm_i915_private *dev_priv) { … } static void g4x_initial_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void g4x_optimize_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } /* latency must be in 0.1us units. */ static unsigned int vlv_wm_method2(unsigned int pixel_rate, unsigned int htotal, unsigned int width, unsigned int cpp, unsigned int latency) { … } static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) { … } static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, int level) { … } static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes) { … } static int vlv_compute_fifo(struct intel_crtc_state *crtc_state) { … } /* mark all levels starting from 'level' as invalid */ static void vlv_invalidate_wms(struct intel_crtc *crtc, struct vlv_wm_state *wm_state, int level) { … } static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size) { … } /* * Starting from 'level' set all higher * levels to 'value' in the "raw" watermarks. */ static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state, int level, enum plane_id plane_id, u16 value) { … } static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { … } static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state, enum plane_id plane_id, int level) { … } static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level) { … } static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state) { … } static int vlv_compute_pipe_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } #define VLV_FIFO … static void vlv_atomic_update_fifo(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } #undef VLV_FIFO static int vlv_compute_intermediate_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void vlv_merge_wm(struct drm_i915_private *dev_priv, struct vlv_wm_values *wm) { … } static void vlv_program_watermarks(struct drm_i915_private *dev_priv) { … } static void vlv_initial_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void vlv_optimize_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void i965_update_wm(struct drm_i915_private *dev_priv) { … } #undef FW_WM static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915, enum i9xx_plane_id i9xx_plane) { … } static void i9xx_update_wm(struct drm_i915_private *dev_priv) { … } static void i845_update_wm(struct drm_i915_private *dev_priv) { … } /* latency must be in 0.1us units. */ static unsigned int ilk_wm_method1(unsigned int pixel_rate, unsigned int cpp, unsigned int latency) { … } /* latency must be in 0.1us units. */ static unsigned int ilk_wm_method2(unsigned int pixel_rate, unsigned int htotal, unsigned int width, unsigned int cpp, unsigned int latency) { … } static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp) { … } struct ilk_wm_maximums { … }; /* * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, u32 mem_value, bool is_lp) { … } /* * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, u32 mem_value) { … } /* * For both WM_PIPE and WM_LP. * mem_value must be in 0.1us units. */ static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, u32 mem_value) { … } /* Only for WM_LP. */ static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state, u32 pri_val) { … } static unsigned int ilk_display_fifo_size(const struct drm_i915_private *dev_priv) { … } static unsigned int ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv, int level, bool is_sprite) { … } static unsigned int ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level) { … } static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv) { … } /* Calculate the maximum primary/sprite plane watermark */ static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, bool is_sprite) { … } /* Calculate the maximum cursor plane watermark */ static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv, int level, const struct intel_wm_config *config) { … } static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, struct ilk_wm_maximums *max) { … } static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv, int level, struct ilk_wm_maximums *max) { … } static bool ilk_validate_wm_level(struct drm_i915_private *i915, int level, const struct ilk_wm_maximums *max, struct intel_wm_level *result) { … } static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, const struct intel_crtc *crtc, int level, struct intel_crtc_state *crtc_state, const struct intel_plane_state *pristate, const struct intel_plane_state *sprstate, const struct intel_plane_state *curstate, struct intel_wm_level *result) { … } static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) { … } static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) { … } static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) { … } static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv, u16 wm[5]) { … } static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv, u16 wm[5]) { … } static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, u16 wm[5], u16 min) { … } static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv) { … } static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv) { … } static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) { … } static bool ilk_validate_pipe_wm(struct drm_i915_private *dev_priv, struct intel_pipe_wm *pipe_wm) { … } /* Compute new watermarks for the pipe */ static int ilk_compute_pipe_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } /* * Build a set of 'intermediate' watermark values that satisfy both the old * state and the new state. These can be programmed to the hardware * immediately. */ static int ilk_compute_intermediate_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } /* * Merge the watermarks from all active pipes for a specific level. */ static void ilk_merge_wm_level(struct drm_i915_private *dev_priv, int level, struct intel_wm_level *ret_wm) { … } /* * Merge all low power watermarks for all active pipes. */ static void ilk_wm_merge(struct drm_i915_private *dev_priv, const struct intel_wm_config *config, const struct ilk_wm_maximums *max, struct intel_pipe_wm *merged) { … } static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) { … } /* The value we need to program into the WM_LPx latency field */ static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv, int level) { … } static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, const struct intel_pipe_wm *merged, enum intel_ddb_partitioning partitioning, struct ilk_wm_values *results) { … } /* * Find the result with the highest level enabled. Check for enable_fbc_wm in * case both are at the same level. Prefer r1 in case they're the same. */ static struct intel_pipe_wm * ilk_find_best_result(struct drm_i915_private *dev_priv, struct intel_pipe_wm *r1, struct intel_pipe_wm *r2) { … } /* dirty bits used to track which watermarks need changes */ #define WM_DIRTY_PIPE(pipe) … #define WM_DIRTY_LP(wm_lp) … #define WM_DIRTY_LP_ALL … #define WM_DIRTY_FBC … #define WM_DIRTY_DDB … static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, const struct ilk_wm_values *old, const struct ilk_wm_values *new) { … } static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, unsigned int dirty) { … } /* * The spec says we shouldn't write when we don't need, because every write * causes WMs to be re-evaluated, expending some power. */ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, struct ilk_wm_values *results) { … } bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv) { … } static void ilk_compute_wm_config(struct drm_i915_private *dev_priv, struct intel_wm_config *config) { … } static void ilk_program_watermarks(struct drm_i915_private *dev_priv) { … } static void ilk_initial_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void ilk_optimize_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { … } static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) { … } static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state) { … } /* * Calculate what we think the watermarks should be for the state we've read * out of the hardware and then immediately program those watermarks so that * we ensure the hardware settings match our internal state. * * We can calculate what we think WM's should be by creating a duplicate of the * current state (which was constructed during hardware readout) and running it * through the atomic check code to calculate new watermark values in the * state object. */ void ilk_wm_sanitize(struct drm_i915_private *dev_priv) { … } #define _FW_WM … #define _FW_WM_VLV … static void g4x_read_wm_values(struct drm_i915_private *dev_priv, struct g4x_wm_values *wm) { … } static void vlv_read_wm_values(struct drm_i915_private *dev_priv, struct vlv_wm_values *wm) { … } #undef _FW_WM #undef _FW_WM_VLV static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) { … } static void g4x_wm_sanitize(struct drm_i915_private *dev_priv) { … } static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915) { … } static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) { … } static void vlv_wm_sanitize(struct drm_i915_private *dev_priv) { … } static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915) { … } /* * FIXME should probably kill this and improve * the real watermark readout/sanitation instead */ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) { … } static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) { … } static const struct intel_wm_funcs ilk_wm_funcs = …; static const struct intel_wm_funcs vlv_wm_funcs = …; static const struct intel_wm_funcs g4x_wm_funcs = …; static const struct intel_wm_funcs pnv_wm_funcs = …; static const struct intel_wm_funcs i965_wm_funcs = …; static const struct intel_wm_funcs i9xx_wm_funcs = …; static const struct intel_wm_funcs i845_wm_funcs = …; static const struct intel_wm_funcs nop_funcs = …; void i9xx_wm_init(struct drm_i915_private *dev_priv) { … }
Codebase Browser
linux