// SPDX-License-Identifier: GPL-2.0 OR MIT
/**************************************************************************
*
* Copyright (c) 2024 Broadcom. All Rights Reserved. The term
* “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sub license, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice (including the
* next paragraph) shall be included in all copies or substantial portions
* of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
* USE OR OTHER DEALINGS IN THE SOFTWARE.
*
**************************************************************************/
#include "vmwgfx_vkms.h"
#include "vmwgfx_bo.h"
#include "vmwgfx_drv.h"
#include "vmwgfx_kms.h"
#include "vmw_surface_cache.h"
#include <drm/drm_crtc.h>
#include <drm/drm_debugfs_crc.h>
#include <drm/drm_print.h>
#include <drm/drm_vblank.h>
#include <linux/crc32.h>
#include <linux/delay.h>
#define GUESTINFO_VBLANK "guestinfo.vmwgfx.vkms_enable"
static int
vmw_surface_sync(struct vmw_private *vmw,
struct vmw_surface *surf)
{
int ret;
struct vmw_fence_obj *fence = NULL;
struct vmw_bo *bo = surf->res.guest_memory_bo;
vmw_resource_clean(&surf->res);
ret = ttm_bo_reserve(&bo->tbo, false, false, NULL);
if (ret != 0) {
drm_warn(&vmw->drm, "%s: failed reserve\n", __func__);
goto done;
}
ret = vmw_execbuf_fence_commands(NULL, vmw, &fence, NULL);
if (ret != 0) {
drm_warn(&vmw->drm, "%s: failed execbuf\n", __func__);
ttm_bo_unreserve(&bo->tbo);
goto done;
}
dma_fence_wait(&fence->base, false);
dma_fence_put(&fence->base);
ttm_bo_unreserve(&bo->tbo);
done:
return ret;
}
static void
compute_crc(struct drm_crtc *crtc,
struct vmw_surface *surf,
u32 *crc)
{
u8 *mapped_surface;
struct vmw_bo *bo = surf->res.guest_memory_bo;
const struct SVGA3dSurfaceDesc *desc =
vmw_surface_get_desc(surf->metadata.format);
u32 row_pitch_bytes;
SVGA3dSize blocks;
u32 y;
*crc = 0;
vmw_surface_get_size_in_blocks(desc, &surf->metadata.base_size, &blocks);
row_pitch_bytes = blocks.width * desc->pitchBytesPerBlock;
WARN_ON(!bo);
mapped_surface = vmw_bo_map_and_cache(bo);
for (y = 0; y < blocks.height; y++) {
*crc = crc32_le(*crc, mapped_surface, row_pitch_bytes);
mapped_surface += row_pitch_bytes;
}
vmw_bo_unmap(bo);
}
static void
crc_generate_worker(struct work_struct *work)
{
struct vmw_display_unit *du =
container_of(work, struct vmw_display_unit, vkms.crc_generator_work);
struct drm_crtc *crtc = &du->crtc;
struct vmw_private *vmw = vmw_priv(crtc->dev);
bool crc_pending;
u64 frame_start, frame_end;
u32 crc32 = 0;
struct vmw_surface *surf = 0;
spin_lock_irq(&du->vkms.crc_state_lock);
crc_pending = du->vkms.crc_pending;
spin_unlock_irq(&du->vkms.crc_state_lock);
/*
* We raced with the vblank hrtimer and previous work already computed
* the crc, nothing to do.
*/
if (!crc_pending)
return;
spin_lock_irq(&du->vkms.crc_state_lock);
surf = vmw_surface_reference(du->vkms.surface);
spin_unlock_irq(&du->vkms.crc_state_lock);
if (surf) {
if (vmw_surface_sync(vmw, surf)) {
drm_warn(
crtc->dev,
"CRC worker wasn't able to sync the crc surface!\n");
return;
}
compute_crc(crtc, surf, &crc32);
vmw_surface_unreference(&surf);
}
spin_lock_irq(&du->vkms.crc_state_lock);
frame_start = du->vkms.frame_start;
frame_end = du->vkms.frame_end;
du->vkms.frame_start = 0;
du->vkms.frame_end = 0;
du->vkms.crc_pending = false;
spin_unlock_irq(&du->vkms.crc_state_lock);
/*
* The worker can fall behind the vblank hrtimer, make sure we catch up.
*/
while (frame_start <= frame_end)
drm_crtc_add_crc_entry(crtc, true, frame_start++, &crc32);
}
static enum hrtimer_restart
vmw_vkms_vblank_simulate(struct hrtimer *timer)
{
struct vmw_display_unit *du = container_of(timer, struct vmw_display_unit, vkms.timer);
struct drm_crtc *crtc = &du->crtc;
struct vmw_private *vmw = vmw_priv(crtc->dev);
bool has_surface = false;
u64 ret_overrun;
bool locked, ret;
ret_overrun = hrtimer_forward_now(&du->vkms.timer,
du->vkms.period_ns);
if (ret_overrun != 1)
drm_dbg_driver(crtc->dev, "vblank timer missed %lld frames.\n",
ret_overrun - 1);
locked = vmw_vkms_vblank_trylock(crtc);
ret = drm_crtc_handle_vblank(crtc);
WARN_ON(!ret);
if (!locked)
return HRTIMER_RESTART;
has_surface = du->vkms.surface != NULL;
vmw_vkms_unlock(crtc);
if (du->vkms.crc_enabled && has_surface) {
u64 frame = drm_crtc_accurate_vblank_count(crtc);
spin_lock(&du->vkms.crc_state_lock);
if (!du->vkms.crc_pending)
du->vkms.frame_start = frame;
else
drm_dbg_driver(crtc->dev,
"crc worker falling behind, frame_start: %llu, frame_end: %llu\n",
du->vkms.frame_start, frame);
du->vkms.frame_end = frame;
du->vkms.crc_pending = true;
spin_unlock(&du->vkms.crc_state_lock);
ret = queue_work(vmw->crc_workq, &du->vkms.crc_generator_work);
if (!ret)
drm_dbg_driver(crtc->dev, "Composer worker already queued\n");
}
return HRTIMER_RESTART;
}
void
vmw_vkms_init(struct vmw_private *vmw)
{
char buffer[64];
const size_t max_buf_len = sizeof(buffer) - 1;
size_t buf_len = max_buf_len;
int ret;
vmw->vkms_enabled = false;
ret = vmw_host_get_guestinfo(GUESTINFO_VBLANK, buffer, &buf_len);
if (ret || buf_len > max_buf_len)
return;
buffer[buf_len] = '\0';
ret = kstrtobool(buffer, &vmw->vkms_enabled);
if (!ret && vmw->vkms_enabled) {
ret = drm_vblank_init(&vmw->drm, VMWGFX_NUM_DISPLAY_UNITS);
vmw->vkms_enabled = (ret == 0);
}
vmw->crc_workq = alloc_ordered_workqueue("vmwgfx_crc_generator", 0);
if (!vmw->crc_workq) {
drm_warn(&vmw->drm, "crc workqueue allocation failed. Disabling vkms.");
vmw->vkms_enabled = false;
}
if (vmw->vkms_enabled)
drm_info(&vmw->drm, "VKMS enabled\n");
}
void
vmw_vkms_cleanup(struct vmw_private *vmw)
{
destroy_workqueue(vmw->crc_workq);
}
bool
vmw_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
int *max_error,
ktime_t *vblank_time,
bool in_vblank_irq)
{
struct drm_device *dev = crtc->dev;
struct vmw_private *vmw = vmw_priv(dev);
unsigned int pipe = crtc->index;
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
if (!vmw->vkms_enabled)
return false;
if (!READ_ONCE(vblank->enabled)) {
*vblank_time = ktime_get();
return true;
}
*vblank_time = READ_ONCE(du->vkms.timer.node.expires);
if (WARN_ON(*vblank_time == vblank->time))
return true;
/*
* To prevent races we roll the hrtimer forward before we do any
* interrupt processing - this is how real hw works (the interrupt is
* only generated after all the vblank registers are updated) and what
* the vblank core expects. Therefore we need to always correct the
* timestampe by one frame.
*/
*vblank_time -= du->vkms.period_ns;
return true;
}
int
vmw_vkms_enable_vblank(struct drm_crtc *crtc)
{
struct drm_device *dev = crtc->dev;
struct vmw_private *vmw = vmw_priv(dev);
unsigned int pipe = drm_crtc_index(crtc);
struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
if (!vmw->vkms_enabled)
return -EINVAL;
drm_calc_timestamping_constants(crtc, &crtc->mode);
hrtimer_init(&du->vkms.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
du->vkms.timer.function = &vmw_vkms_vblank_simulate;
du->vkms.period_ns = ktime_set(0, vblank->framedur_ns);
hrtimer_start(&du->vkms.timer, du->vkms.period_ns, HRTIMER_MODE_REL);
return 0;
}
void
vmw_vkms_disable_vblank(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (!vmw->vkms_enabled)
return;
hrtimer_cancel(&du->vkms.timer);
du->vkms.surface = NULL;
du->vkms.period_ns = ktime_set(0, 0);
}
enum vmw_vkms_lock_state {
VMW_VKMS_LOCK_UNLOCKED = 0,
VMW_VKMS_LOCK_MODESET = 1,
VMW_VKMS_LOCK_VBLANK = 2
};
void
vmw_vkms_crtc_init(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
atomic_set(&du->vkms.atomic_lock, VMW_VKMS_LOCK_UNLOCKED);
spin_lock_init(&du->vkms.crc_state_lock);
INIT_WORK(&du->vkms.crc_generator_work, crc_generate_worker);
du->vkms.surface = NULL;
}
void
vmw_vkms_crtc_cleanup(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
if (du->vkms.surface)
vmw_surface_unreference(&du->vkms.surface);
WARN_ON(work_pending(&du->vkms.crc_generator_work));
hrtimer_cancel(&du->vkms.timer);
}
void
vmw_vkms_crtc_atomic_begin(struct drm_crtc *crtc,
struct drm_atomic_state *state)
{
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (vmw->vkms_enabled)
vmw_vkms_modeset_lock(crtc);
}
void
vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
struct drm_atomic_state *state)
{
unsigned long flags;
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (!vmw->vkms_enabled)
return;
if (crtc->state->event) {
spin_lock_irqsave(&crtc->dev->event_lock, flags);
if (drm_crtc_vblank_get(crtc) != 0)
drm_crtc_send_vblank_event(crtc, crtc->state->event);
else
drm_crtc_arm_vblank_event(crtc, crtc->state->event);
spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
crtc->state->event = NULL;
}
vmw_vkms_unlock(crtc);
}
void
vmw_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
struct drm_atomic_state *state)
{
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (vmw->vkms_enabled)
drm_crtc_vblank_on(crtc);
}
void
vmw_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
struct drm_atomic_state *state)
{
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (vmw->vkms_enabled)
drm_crtc_vblank_off(crtc);
}
static bool
is_crc_supported(struct drm_crtc *crtc)
{
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (!vmw->vkms_enabled)
return false;
if (vmw->active_display_unit != vmw_du_screen_target)
return false;
return true;
}
static const char * const pipe_crc_sources[] = {"auto"};
static int
crc_parse_source(const char *src_name,
bool *enabled)
{
int ret = 0;
if (!src_name) {
*enabled = false;
} else if (strcmp(src_name, "auto") == 0) {
*enabled = true;
} else {
*enabled = false;
ret = -EINVAL;
}
return ret;
}
const char *const *
vmw_vkms_get_crc_sources(struct drm_crtc *crtc,
size_t *count)
{
*count = 0;
if (!is_crc_supported(crtc))
return NULL;
*count = ARRAY_SIZE(pipe_crc_sources);
return pipe_crc_sources;
}
int
vmw_vkms_verify_crc_source(struct drm_crtc *crtc,
const char *src_name,
size_t *values_cnt)
{
bool enabled;
if (!is_crc_supported(crtc))
return -EINVAL;
if (crc_parse_source(src_name, &enabled) < 0) {
drm_dbg_driver(crtc->dev, "unknown source '%s'\n", src_name);
return -EINVAL;
}
*values_cnt = 1;
return 0;
}
int
vmw_vkms_set_crc_source(struct drm_crtc *crtc,
const char *src_name)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
bool enabled, prev_enabled, locked;
int ret;
if (!is_crc_supported(crtc))
return -EINVAL;
ret = crc_parse_source(src_name, &enabled);
if (enabled)
drm_crtc_vblank_get(crtc);
locked = vmw_vkms_modeset_lock_relaxed(crtc);
prev_enabled = du->vkms.crc_enabled;
du->vkms.crc_enabled = enabled;
if (locked)
vmw_vkms_unlock(crtc);
if (prev_enabled)
drm_crtc_vblank_put(crtc);
return ret;
}
void
vmw_vkms_set_crc_surface(struct drm_crtc *crtc,
struct vmw_surface *surf)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
struct vmw_private *vmw = vmw_priv(crtc->dev);
if (vmw->vkms_enabled && du->vkms.surface != surf) {
WARN_ON(atomic_read(&du->vkms.atomic_lock) != VMW_VKMS_LOCK_MODESET);
if (du->vkms.surface)
vmw_surface_unreference(&du->vkms.surface);
if (surf)
du->vkms.surface = vmw_surface_reference(surf);
}
}
/**
* vmw_vkms_lock_max_wait_ns - Return the max wait for the vkms lock
* @du: The vmw_display_unit from which to grab the vblank timings
*
* Returns the maximum wait time used to acquire the vkms lock. By
* default uses a time of a single frame and in case where vblank
* was not initialized for the display unit 1/60th of a second.
*/
static inline u64
vmw_vkms_lock_max_wait_ns(struct vmw_display_unit *du)
{
s64 nsecs = ktime_to_ns(du->vkms.period_ns);
return (nsecs > 0) ? nsecs : 16666666;
}
/**
* vmw_vkms_modeset_lock - Protects access to crtc during modeset
* @crtc: The crtc to lock for vkms
*
* This function prevents the VKMS timers/callbacks from being called
* while a modeset operation is in process. We don't want the callbacks
* e.g. the vblank simulator to be trying to access incomplete state
* so we need to make sure they execute only when the modeset has
* finished.
*
* Normally this would have been done with a spinlock but locking the
* entire atomic modeset with vmwgfx is impossible because kms prepare
* executes non-atomic ops (e.g. vmw_validation_prepare holds a mutex to
* guard various bits of state). Which means that we need to synchronize
* atomic context (the vblank handler) with the non-atomic entirity
* of kms - so use an atomic_t to track which part of vkms has access
* to the basic vkms state.
*/
void
vmw_vkms_modeset_lock(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
const u64 nsecs_delay = 10;
const u64 MAX_NSECS_DELAY = vmw_vkms_lock_max_wait_ns(du);
u64 total_delay = 0;
int ret;
do {
ret = atomic_cmpxchg(&du->vkms.atomic_lock,
VMW_VKMS_LOCK_UNLOCKED,
VMW_VKMS_LOCK_MODESET);
if (ret == VMW_VKMS_LOCK_UNLOCKED || total_delay >= MAX_NSECS_DELAY)
break;
ndelay(nsecs_delay);
total_delay += nsecs_delay;
} while (1);
if (total_delay >= MAX_NSECS_DELAY) {
drm_warn(crtc->dev, "VKMS lock expired! total_delay = %lld, ret = %d, cur = %d\n",
total_delay, ret, atomic_read(&du->vkms.atomic_lock));
}
}
/**
* vmw_vkms_modeset_lock_relaxed - Protects access to crtc during modeset
* @crtc: The crtc to lock for vkms
*
* Much like vmw_vkms_modeset_lock except that when the crtc is currently
* in a modeset it will return immediately.
*
* Returns true if actually locked vkms to modeset or false otherwise.
*/
bool
vmw_vkms_modeset_lock_relaxed(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
const u64 nsecs_delay = 10;
const u64 MAX_NSECS_DELAY = vmw_vkms_lock_max_wait_ns(du);
u64 total_delay = 0;
int ret;
do {
ret = atomic_cmpxchg(&du->vkms.atomic_lock,
VMW_VKMS_LOCK_UNLOCKED,
VMW_VKMS_LOCK_MODESET);
if (ret == VMW_VKMS_LOCK_UNLOCKED ||
ret == VMW_VKMS_LOCK_MODESET ||
total_delay >= MAX_NSECS_DELAY)
break;
ndelay(nsecs_delay);
total_delay += nsecs_delay;
} while (1);
if (total_delay >= MAX_NSECS_DELAY) {
drm_warn(crtc->dev, "VKMS relaxed lock expired!\n");
return false;
}
return ret == VMW_VKMS_LOCK_UNLOCKED;
}
/**
* vmw_vkms_vblank_trylock - Protects access to crtc during vblank
* @crtc: The crtc to lock for vkms
*
* Tries to lock vkms for vblank, returns immediately.
*
* Returns true if locked vkms to vblank or false otherwise.
*/
bool
vmw_vkms_vblank_trylock(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
u32 ret;
ret = atomic_cmpxchg(&du->vkms.atomic_lock,
VMW_VKMS_LOCK_UNLOCKED,
VMW_VKMS_LOCK_VBLANK);
return ret == VMW_VKMS_LOCK_UNLOCKED;
}
void
vmw_vkms_unlock(struct drm_crtc *crtc)
{
struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
/* Release flag; mark it as unlocked. */
atomic_set(&du->vkms.atomic_lock, VMW_VKMS_LOCK_UNLOCKED);
}
Codebase Browser
linux