// SPDX-License-Identifier: MIT
/*
* Copyright © 2022-2023 Intel Corporation
*/
#include "i915_drv.h"
#include "i915_reg.h"
#include "intel_color.h"
#include "intel_crtc.h"
#include "intel_de.h"
#include "intel_display_types.h"
#include "intel_vblank.h"
#include "intel_vrr.h"
/*
* This timing diagram depicts the video signal in and
* around the vertical blanking period.
*
* Assumptions about the fictitious mode used in this example:
* vblank_start >= 3
* vsync_start = vblank_start + 1
* vsync_end = vblank_start + 2
* vtotal = vblank_start + 3
*
* start of vblank:
* latch double buffered registers
* increment frame counter (ctg+)
* generate start of vblank interrupt (gen4+)
* |
* | frame start:
* | generate frame start interrupt (aka. vblank interrupt) (gmch)
* | may be shifted forward 1-3 extra lines via TRANSCONF
* | |
* | | start of vsync:
* | | generate vsync interrupt
* | | |
* ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
* . \hs/ . \hs/ \hs/ \hs/ . \hs/
* ----va---> <-----------------vb--------------------> <--------va-------------
* | | <----vs-----> |
* -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
* -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
* -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
* | | |
* last visible pixel first visible pixel
* | increment frame counter (gen3/4)
* pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
*
* x = horizontal active
* _ = horizontal blanking
* hs = horizontal sync
* va = vertical active
* vb = vertical blanking
* vs = vertical sync
* vbs = vblank_start (number)
*
* Summary:
* - most events happen at the start of horizontal sync
* - frame start happens at the start of horizontal blank, 1-4 lines
* (depending on TRANSCONF settings) after the start of vblank
* - gen3/4 pixel and frame counter are synchronized with the start
* of horizontal active on the first line of vertical active
*/
/*
* Called from drm generic code, passed a 'crtc', which we use as a pipe index.
*/
u32 i915_get_vblank_counter(struct drm_crtc *crtc)
{
struct intel_display *display = to_intel_display(crtc->dev);
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
const struct drm_display_mode *mode = &vblank->hwmode;
enum pipe pipe = to_intel_crtc(crtc)->pipe;
u32 pixel, vbl_start, hsync_start, htotal;
u64 frame;
/*
* On i965gm TV output the frame counter only works up to
* the point when we enable the TV encoder. After that the
* frame counter ceases to work and reads zero. We need a
* vblank wait before enabling the TV encoder and so we
* have to enable vblank interrupts while the frame counter
* is still in a working state. However the core vblank code
* does not like us returning non-zero frame counter values
* when we've told it that we don't have a working frame
* counter. Thus we must stop non-zero values leaking out.
*/
if (!vblank->max_vblank_count)
return 0;
htotal = mode->crtc_htotal;
hsync_start = mode->crtc_hsync_start;
vbl_start = intel_mode_vblank_start(mode);
/* Convert to pixel count */
vbl_start *= htotal;
/* Start of vblank event occurs at start of hsync */
vbl_start -= htotal - hsync_start;
/*
* High & low register fields aren't synchronized, so make sure
* we get a low value that's stable across two reads of the high
* register.
*/
frame = intel_de_read64_2x32(display, PIPEFRAMEPIXEL(display, pipe),
PIPEFRAME(display, pipe));
pixel = frame & PIPE_PIXEL_MASK;
frame = (frame >> PIPE_FRAME_LOW_SHIFT) & 0xffffff;
/*
* The frame counter increments at beginning of active.
* Cook up a vblank counter by also checking the pixel
* counter against vblank start.
*/
return (frame + (pixel >= vbl_start)) & 0xffffff;
}
u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
{
struct intel_display *display = to_intel_display(crtc->dev);
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
enum pipe pipe = to_intel_crtc(crtc)->pipe;
if (!vblank->max_vblank_count)
return 0;
return intel_de_read(display, PIPE_FRMCOUNT_G4X(display, pipe));
}
static u32 intel_crtc_scanlines_since_frame_timestamp(struct intel_crtc *crtc)
{
struct intel_display *display = to_intel_display(crtc);
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
const struct drm_display_mode *mode = &vblank->hwmode;
u32 htotal = mode->crtc_htotal;
u32 clock = mode->crtc_clock;
u32 scan_prev_time, scan_curr_time, scan_post_time;
/*
* To avoid the race condition where we might cross into the
* next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
* reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
* during the same frame.
*/
do {
/*
* This field provides read back of the display
* pipe frame time stamp. The time stamp value
* is sampled at every start of vertical blank.
*/
scan_prev_time = intel_de_read_fw(display,
PIPE_FRMTMSTMP(crtc->pipe));
/*
* The TIMESTAMP_CTR register has the current
* time stamp value.
*/
scan_curr_time = intel_de_read_fw(display, IVB_TIMESTAMP_CTR);
scan_post_time = intel_de_read_fw(display,
PIPE_FRMTMSTMP(crtc->pipe));
} while (scan_post_time != scan_prev_time);
return div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
clock), 1000 * htotal);
}
/*
* On certain encoders on certain platforms, pipe
* scanline register will not work to get the scanline,
* since the timings are driven from the PORT or issues
* with scanline register updates.
* This function will use Framestamp and current
* timestamp registers to calculate the scanline.
*/
static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
{
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
const struct drm_display_mode *mode = &vblank->hwmode;
u32 vblank_start = mode->crtc_vblank_start;
u32 vtotal = mode->crtc_vtotal;
u32 scanline;
scanline = intel_crtc_scanlines_since_frame_timestamp(crtc);
scanline = min(scanline, vtotal - 1);
scanline = (scanline + vblank_start) % vtotal;
return scanline;
}
int intel_crtc_scanline_offset(const struct intel_crtc_state *crtc_state)
{
struct intel_display *display = to_intel_display(crtc_state);
struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
/*
* The scanline counter increments at the leading edge of hsync.
*
* On most platforms it starts counting from vtotal-1 on the
* first active line. That means the scanline counter value is
* always one less than what we would expect. Ie. just after
* start of vblank, which also occurs at start of hsync (on the
* last active line), the scanline counter will read vblank_start-1.
*
* On gen2 the scanline counter starts counting from 1 instead
* of vtotal-1, so we have to subtract one.
*
* On HSW+ the behaviour of the scanline counter depends on the output
* type. For DP ports it behaves like most other platforms, but on HDMI
* there's an extra 1 line difference. So we need to add two instead of
* one to the value.
*
* On VLV/CHV DSI the scanline counter would appear to increment
* approx. 1/3 of a scanline before start of vblank. Unfortunately
* that means we can't tell whether we're in vblank or not while
* we're on that particular line. We must still set scanline_offset
* to 1 so that the vblank timestamps come out correct when we query
* the scanline counter from within the vblank interrupt handler.
* However if queried just before the start of vblank we'll get an
* answer that's slightly in the future.
*/
if (DISPLAY_VER(display) == 2)
return -1;
else if (HAS_DDI(i915) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
return 2;
else
return 1;
}
/*
* intel_de_read_fw(), only for fast reads of display block, no need for
* forcewake etc.
*/
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
struct intel_display *display = to_intel_display(crtc);
struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
const struct drm_display_mode *mode = &vblank->hwmode;
enum pipe pipe = crtc->pipe;
int position, vtotal;
if (!crtc->active)
return 0;
if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
return __intel_get_crtc_scanline_from_timestamp(crtc);
vtotal = intel_mode_vtotal(mode);
position = intel_de_read_fw(display, PIPEDSL(display, pipe)) & PIPEDSL_LINE_MASK;
/*
* On HSW, the DSL reg (0x70000) appears to return 0 if we
* read it just before the start of vblank. So try it again
* so we don't accidentally end up spanning a vblank frame
* increment, causing the pipe_update_end() code to squak at us.
*
* The nature of this problem means we can't simply check the ISR
* bit and return the vblank start value; nor can we use the scanline
* debug register in the transcoder as it appears to have the same
* problem. We may need to extend this to include other platforms,
* but so far testing only shows the problem on HSW.
*/
if (HAS_DDI(display) && !position) {
int i, temp;
for (i = 0; i < 100; i++) {
udelay(1);
temp = intel_de_read_fw(display,
PIPEDSL(display, pipe)) & PIPEDSL_LINE_MASK;
if (temp != position) {
position = temp;
break;
}
}
}
/*
* See update_scanline_offset() for the details on the
* scanline_offset adjustment.
*/
return (position + vtotal + crtc->scanline_offset) % vtotal;
}
/*
* The uncore version of the spin lock functions is used to decide
* whether we need to lock the uncore lock or not. This is only
* needed in i915, not in Xe.
*
* This lock in i915 is needed because some old platforms (at least
* IVB and possibly HSW as well), which are not supported in Xe, need
* all register accesses to the same cacheline to be serialized,
* otherwise they may hang.
*/
#ifdef I915
static void intel_vblank_section_enter(struct intel_display *display)
__acquires(i915->uncore.lock)
{
struct drm_i915_private *i915 = to_i915(display->drm);
spin_lock(&i915->uncore.lock);
}
static void intel_vblank_section_exit(struct intel_display *display)
__releases(i915->uncore.lock)
{
struct drm_i915_private *i915 = to_i915(display->drm);
spin_unlock(&i915->uncore.lock);
}
#else
static void intel_vblank_section_enter(struct intel_display *display)
{
}
static void intel_vblank_section_exit(struct intel_display *display)
{
}
#endif
static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
bool in_vblank_irq,
int *vpos, int *hpos,
ktime_t *stime, ktime_t *etime,
const struct drm_display_mode *mode)
{
struct intel_display *display = to_intel_display(_crtc->dev);
struct drm_i915_private *dev_priv = to_i915(display->drm);
struct intel_crtc *crtc = to_intel_crtc(_crtc);
enum pipe pipe = crtc->pipe;
int position;
int vbl_start, vbl_end, hsync_start, htotal, vtotal;
unsigned long irqflags;
bool use_scanline_counter = DISPLAY_VER(display) >= 5 ||
IS_G4X(dev_priv) || DISPLAY_VER(display) == 2 ||
crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
if (drm_WARN_ON(display->drm, !mode->crtc_clock)) {
drm_dbg(display->drm,
"trying to get scanoutpos for disabled pipe %c\n",
pipe_name(pipe));
return false;
}
htotal = mode->crtc_htotal;
hsync_start = mode->crtc_hsync_start;
vtotal = intel_mode_vtotal(mode);
vbl_start = intel_mode_vblank_start(mode);
vbl_end = intel_mode_vblank_end(mode);
/*
* Enter vblank critical section, as we will do multiple
* timing critical raw register reads, potentially with
* preemption disabled, so the following code must not block.
*/
local_irq_save(irqflags);
intel_vblank_section_enter(display);
/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
/* Get optional system timestamp before query. */
if (stime)
*stime = ktime_get();
if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
int scanlines = intel_crtc_scanlines_since_frame_timestamp(crtc);
position = __intel_get_crtc_scanline(crtc);
/*
* Already exiting vblank? If so, shift our position
* so it looks like we're already apporaching the full
* vblank end. This should make the generated timestamp
* more or less match when the active portion will start.
*/
if (position >= vbl_start && scanlines < position)
position = min(crtc->vmax_vblank_start + scanlines, vtotal - 1);
} else if (use_scanline_counter) {
/* No obvious pixelcount register. Only query vertical
* scanout position from Display scan line register.
*/
position = __intel_get_crtc_scanline(crtc);
} else {
/*
* Have access to pixelcount since start of frame.
* We can split this into vertical and horizontal
* scanout position.
*/
position = (intel_de_read_fw(display, PIPEFRAMEPIXEL(display, pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
/* convert to pixel counts */
vbl_start *= htotal;
vbl_end *= htotal;
vtotal *= htotal;
/*
* In interlaced modes, the pixel counter counts all pixels,
* so one field will have htotal more pixels. In order to avoid
* the reported position from jumping backwards when the pixel
* counter is beyond the length of the shorter field, just
* clamp the position the length of the shorter field. This
* matches how the scanline counter based position works since
* the scanline counter doesn't count the two half lines.
*/
position = min(position, vtotal - 1);
/*
* Start of vblank interrupt is triggered at start of hsync,
* just prior to the first active line of vblank. However we
* consider lines to start at the leading edge of horizontal
* active. So, should we get here before we've crossed into
* the horizontal active of the first line in vblank, we would
* not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
* always add htotal-hsync_start to the current pixel position.
*/
position = (position + htotal - hsync_start) % vtotal;
}
/* Get optional system timestamp after query. */
if (etime)
*etime = ktime_get();
/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
intel_vblank_section_exit(display);
local_irq_restore(irqflags);
/*
* While in vblank, position will be negative
* counting up towards 0 at vbl_end. And outside
* vblank, position will be positive counting
* up since vbl_end.
*/
if (position >= vbl_start)
position -= vbl_end;
else
position += vtotal - vbl_end;
if (use_scanline_counter) {
*vpos = position;
*hpos = 0;
} else {
*vpos = position / htotal;
*hpos = position - (*vpos * htotal);
}
return true;
}
bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
ktime_t *vblank_time, bool in_vblank_irq)
{
return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
crtc, max_error, vblank_time, in_vblank_irq,
i915_get_crtc_scanoutpos);
}
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
struct intel_display *display = to_intel_display(crtc);
unsigned long irqflags;
int position;
local_irq_save(irqflags);
intel_vblank_section_enter(display);
position = __intel_get_crtc_scanline(crtc);
intel_vblank_section_exit(display);
local_irq_restore(irqflags);
return position;
}
static bool pipe_scanline_is_moving(struct intel_display *display,
enum pipe pipe)
{
i915_reg_t reg = PIPEDSL(display, pipe);
u32 line1, line2;
line1 = intel_de_read(display, reg) & PIPEDSL_LINE_MASK;
msleep(5);
line2 = intel_de_read(display, reg) & PIPEDSL_LINE_MASK;
return line1 != line2;
}
static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
{
struct intel_display *display = to_intel_display(crtc);
enum pipe pipe = crtc->pipe;
/* Wait for the display line to settle/start moving */
if (wait_for(pipe_scanline_is_moving(display, pipe) == state, 100))
drm_err(display->drm,
"pipe %c scanline %s wait timed out\n",
pipe_name(pipe), str_on_off(state));
}
void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
{
wait_for_pipe_scanline_moving(crtc, false);
}
void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
{
wait_for_pipe_scanline_moving(crtc, true);
}
void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state,
bool vrr_enable)
{
struct intel_display *display = to_intel_display(crtc_state);
struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
u8 mode_flags = crtc_state->mode_flags;
struct drm_display_mode adjusted_mode;
int vmax_vblank_start = 0;
unsigned long irqflags;
drm_mode_init(&adjusted_mode, &crtc_state->hw.adjusted_mode);
if (vrr_enable) {
drm_WARN_ON(display->drm,
(mode_flags & I915_MODE_FLAG_VRR) == 0);
adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax;
adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax;
adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
} else {
mode_flags &= ~I915_MODE_FLAG_VRR;
}
/*
* Belts and suspenders locking to guarantee everyone sees 100%
* consistent state during fastset seamless refresh rate changes.
*
* vblank_time_lock takes care of all drm_vblank.c stuff, and
* uncore.lock takes care of __intel_get_crtc_scanline() which
* may get called elsewhere as well.
*
* TODO maybe just protect everything (including
* __intel_get_crtc_scanline()) with vblank_time_lock?
* Need to audit everything to make sure it's safe.
*/
spin_lock_irqsave(&display->drm->vblank_time_lock, irqflags);
intel_vblank_section_enter(display);
drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);
crtc->vmax_vblank_start = vmax_vblank_start;
crtc->mode_flags = mode_flags;
crtc->scanline_offset = intel_crtc_scanline_offset(crtc_state);
intel_vblank_section_exit(display);
spin_unlock_irqrestore(&display->drm->vblank_time_lock, irqflags);
}
int intel_mode_vdisplay(const struct drm_display_mode *mode)
{
int vdisplay = mode->crtc_vdisplay;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
vdisplay = DIV_ROUND_UP(vdisplay, 2);
return vdisplay;
}
int intel_mode_vblank_start(const struct drm_display_mode *mode)
{
int vblank_start = mode->crtc_vblank_start;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
vblank_start = DIV_ROUND_UP(vblank_start, 2);
return vblank_start;
}
int intel_mode_vblank_end(const struct drm_display_mode *mode)
{
int vblank_end = mode->crtc_vblank_end;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
vblank_end /= 2;
return vblank_end;
}
int intel_mode_vtotal(const struct drm_display_mode *mode)
{
int vtotal = mode->crtc_vtotal;
if (mode->flags & DRM_MODE_FLAG_INTERLACE)
vtotal /= 2;
return vtotal;
}
void intel_vblank_evade_init(const struct intel_crtc_state *old_crtc_state,
const struct intel_crtc_state *new_crtc_state,
struct intel_vblank_evade_ctx *evade)
{
struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
struct drm_i915_private *i915 = to_i915(crtc->base.dev);
const struct intel_crtc_state *crtc_state;
const struct drm_display_mode *adjusted_mode;
evade->crtc = crtc;
evade->need_vlv_dsi_wa = (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) &&
intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
/*
* During fastsets/etc. the transcoder is still
* running with the old timings at this point.
*
* TODO: maybe just use the active timings here?
*/
if (intel_crtc_needs_modeset(new_crtc_state))
crtc_state = new_crtc_state;
else
crtc_state = old_crtc_state;
adjusted_mode = &crtc_state->hw.adjusted_mode;
if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
/* timing changes should happen with VRR disabled */
drm_WARN_ON(crtc->base.dev, intel_crtc_needs_modeset(new_crtc_state) ||
new_crtc_state->update_m_n || new_crtc_state->update_lrr);
if (intel_vrr_is_push_sent(crtc_state))
evade->vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
else
evade->vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
} else {
evade->vblank_start = intel_mode_vblank_start(adjusted_mode);
}
/* FIXME needs to be calibrated sensibly */
evade->min = evade->vblank_start - intel_usecs_to_scanlines(adjusted_mode,
VBLANK_EVASION_TIME_US);
evade->max = evade->vblank_start - 1;
/*
* M/N and TRANS_VTOTAL are double buffered on the transcoder's
* undelayed vblank, so with seamless M/N and LRR we must evade
* both vblanks.
*
* DSB execution waits for the transcoder's undelayed vblank,
* hence we must kick off the commit before that.
*/
if (intel_color_uses_dsb(new_crtc_state) ||
new_crtc_state->update_m_n || new_crtc_state->update_lrr)
evade->min -= intel_mode_vblank_start(adjusted_mode) -
intel_mode_vdisplay(adjusted_mode);
}
/* must be called with vblank interrupt already enabled! */
int intel_vblank_evade(struct intel_vblank_evade_ctx *evade)
{
struct intel_crtc *crtc = evade->crtc;
struct intel_display *display = to_intel_display(crtc);
long timeout = msecs_to_jiffies_timeout(1);
wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
DEFINE_WAIT(wait);
int scanline;
if (evade->min <= 0 || evade->max <= 0)
return 0;
for (;;) {
/*
* prepare_to_wait() has a memory barrier, which guarantees
* other CPUs can see the task state update by the time we
* read the scanline.
*/
prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
scanline = intel_get_crtc_scanline(crtc);
if (scanline < evade->min || scanline > evade->max)
break;
if (!timeout) {
drm_err(display->drm,
"Potential atomic update failure on pipe %c\n",
pipe_name(crtc->pipe));
break;
}
local_irq_enable();
timeout = schedule_timeout(timeout);
local_irq_disable();
}
finish_wait(wq, &wait);
/*
* On VLV/CHV DSI the scanline counter would appear to
* increment approx. 1/3 of a scanline before start of vblank.
* The registers still get latched at start of vblank however.
* This means we must not write any registers on the first
* line of vblank (since not the whole line is actually in
* vblank). And unfortunately we can't use the interrupt to
* wait here since it will fire too soon. We could use the
* frame start interrupt instead since it will fire after the
* critical scanline, but that would require more changes
* in the interrupt code. So for now we'll just do the nasty
* thing and poll for the bad scanline to pass us by.
*
* FIXME figure out if BXT+ DSI suffers from this as well
*/
while (evade->need_vlv_dsi_wa && scanline == evade->vblank_start)
scanline = intel_get_crtc_scanline(crtc);
return scanline;
}
Codebase Browser
linux