/*
* SPDX-License-Identifier: MIT
*
* Copyright © 2018 Intel Corporation
*/
#include <linux/kref.h>
#include <linux/string_helpers.h>
#include "gem/i915_gem_pm.h"
#include "gt/intel_gt.h"
#include "i915_selftest.h"
#include "igt_flush_test.h"
#include "lib_sw_fence.h"
struct live_active {
struct i915_active base;
struct kref ref;
bool retired;
};
static void __live_get(struct live_active *active)
{
kref_get(&active->ref);
}
static void __live_free(struct live_active *active)
{
i915_active_fini(&active->base);
kfree(active);
}
static void __live_release(struct kref *ref)
{
struct live_active *active = container_of(ref, typeof(*active), ref);
__live_free(active);
}
static void __live_put(struct live_active *active)
{
kref_put(&active->ref, __live_release);
}
static int __live_active(struct i915_active *base)
{
struct live_active *active = container_of(base, typeof(*active), base);
__live_get(active);
return 0;
}
static void __live_retire(struct i915_active *base)
{
struct live_active *active = container_of(base, typeof(*active), base);
active->retired = true;
__live_put(active);
}
static struct live_active *__live_alloc(struct drm_i915_private *i915)
{
struct live_active *active;
active = kzalloc(sizeof(*active), GFP_KERNEL);
if (!active)
return NULL;
kref_init(&active->ref);
i915_active_init(&active->base, __live_active, __live_retire, 0);
return active;
}
static struct live_active *
__live_active_setup(struct drm_i915_private *i915)
{
struct intel_engine_cs *engine;
struct i915_sw_fence *submit;
struct live_active *active;
unsigned int count = 0;
int err = 0;
active = __live_alloc(i915);
if (!active)
return ERR_PTR(-ENOMEM);
submit = heap_fence_create(GFP_KERNEL);
if (!submit) {
kfree(active);
return ERR_PTR(-ENOMEM);
}
err = i915_active_acquire(&active->base);
if (err)
goto out;
for_each_uabi_engine(engine, i915) {
struct i915_request *rq;
rq = intel_engine_create_kernel_request(engine);
if (IS_ERR(rq)) {
err = PTR_ERR(rq);
break;
}
err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
submit,
GFP_KERNEL);
if (err >= 0)
err = i915_active_add_request(&active->base, rq);
i915_request_add(rq);
if (err) {
pr_err("Failed to track active ref!\n");
break;
}
count++;
}
i915_active_release(&active->base);
if (READ_ONCE(active->retired) && count) {
pr_err("i915_active retired before submission!\n");
err = -EINVAL;
}
if (atomic_read(&active->base.count) != count) {
pr_err("i915_active not tracking all requests, found %d, expected %d\n",
atomic_read(&active->base.count), count);
err = -EINVAL;
}
out:
i915_sw_fence_commit(submit);
heap_fence_put(submit);
if (err) {
__live_put(active);
active = ERR_PTR(err);
}
return active;
}
static int live_active_wait(void *arg)
{
struct drm_i915_private *i915 = arg;
struct live_active *active;
int err = 0;
/* Check that we get a callback when requests retire upon waiting */
active = __live_active_setup(i915);
if (IS_ERR(active))
return PTR_ERR(active);
__i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
if (!READ_ONCE(active->retired)) {
struct drm_printer p = drm_err_printer(&i915->drm, __func__);
drm_printf(&p, "i915_active not retired after waiting!\n");
i915_active_print(&active->base, &p);
err = -EINVAL;
}
__live_put(active);
if (igt_flush_test(i915))
err = -EIO;
return err;
}
static int live_active_retire(void *arg)
{
struct drm_i915_private *i915 = arg;
struct live_active *active;
int err = 0;
/* Check that we get a callback when requests are indirectly retired */
active = __live_active_setup(i915);
if (IS_ERR(active))
return PTR_ERR(active);
/* waits for & retires all requests */
if (igt_flush_test(i915))
err = -EIO;
if (!READ_ONCE(active->retired)) {
struct drm_printer p = drm_err_printer(&i915->drm, __func__);
drm_printf(&p, "i915_active not retired after flushing!\n");
i915_active_print(&active->base, &p);
err = -EINVAL;
}
__live_put(active);
return err;
}
static int live_active_barrier(void *arg)
{
struct drm_i915_private *i915 = arg;
struct intel_engine_cs *engine;
struct live_active *active;
int err = 0;
/* Check that we get a callback when requests retire upon waiting */
active = __live_alloc(i915);
if (!active)
return -ENOMEM;
err = i915_active_acquire(&active->base);
if (err)
goto out;
for_each_uabi_engine(engine, i915) {
err = i915_active_acquire_preallocate_barrier(&active->base,
engine);
if (err)
break;
i915_active_acquire_barrier(&active->base);
}
i915_active_release(&active->base);
if (err)
goto out;
__i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
if (!READ_ONCE(active->retired)) {
pr_err("i915_active not retired after flushing barriers!\n");
err = -EINVAL;
}
out:
__live_put(active);
if (igt_flush_test(i915))
err = -EIO;
return err;
}
int i915_active_live_selftests(struct drm_i915_private *i915)
{
static const struct i915_subtest tests[] = {
SUBTEST(live_active_wait),
SUBTEST(live_active_retire),
SUBTEST(live_active_barrier),
};
if (intel_gt_is_wedged(to_gt(i915)))
return 0;
return i915_subtests(tests, i915);
}
static struct intel_engine_cs *node_to_barrier(struct active_node *it)
{
struct intel_engine_cs *engine;
if (!is_barrier(&it->base))
return NULL;
engine = __barrier_to_engine(it);
smp_rmb(); /* serialise with add_active_barriers */
if (!is_barrier(&it->base))
return NULL;
return engine;
}
void i915_active_print(struct i915_active *ref, struct drm_printer *m)
{
drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire);
drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count));
drm_printf(m, "\tpreallocated barriers? %s\n",
str_yes_no(!llist_empty(&ref->preallocated_barriers)));
if (i915_active_acquire_if_busy(ref)) {
struct active_node *it, *n;
rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
struct intel_engine_cs *engine;
engine = node_to_barrier(it);
if (engine) {
drm_printf(m, "\tbarrier: %s\n", engine->name);
continue;
}
if (i915_active_fence_isset(&it->base)) {
drm_printf(m,
"\ttimeline: %llx\n", it->timeline);
continue;
}
}
i915_active_release(ref);
}
}
static void spin_unlock_wait(spinlock_t *lock)
{
spin_lock_irq(lock);
spin_unlock_irq(lock);
}
static void active_flush(struct i915_active *ref,
struct i915_active_fence *active)
{
struct dma_fence *fence;
fence = xchg(__active_fence_slot(active), NULL);
if (!fence)
return;
spin_lock_irq(fence->lock);
__list_del_entry(&active->cb.node);
spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */
atomic_dec(&ref->count);
GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
}
void i915_active_unlock_wait(struct i915_active *ref)
{
if (i915_active_acquire_if_busy(ref)) {
struct active_node *it, *n;
/* Wait for all active callbacks */
rcu_read_lock();
active_flush(ref, &ref->excl);
rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node)
active_flush(ref, &it->base);
rcu_read_unlock();
i915_active_release(ref);
}
/* And wait for the retire callback */
spin_unlock_wait(&ref->tree_lock);
/* ... which may have been on a thread instead */
flush_work(&ref->work);
}
Codebase Browser
linux