linux/drivers/gpu/drm/i915/i915_sw_fence.c

/*
 * SPDX-License-Identifier: MIT
 *
 * (C) Copyright 2016 Intel Corporation
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/irq_work.h>
#include <linux/dma-resv.h>

#include "i915_sw_fence.h"
#include "i915_selftest.h"

#if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
#define I915_SW_FENCE_BUG_ON …
#else
#define I915_SW_FENCE_BUG_ON(expr) …
#endif

#ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
static DEFINE_SPINLOCK(i915_sw_fence_lock);
#endif

#define WQ_FLAG_BITS …

/* after WQ_FLAG_* for safety */
#define I915_SW_FENCE_FLAG_FENCE …
#define I915_SW_FENCE_FLAG_ALLOC …

enum { … };

static void *i915_sw_fence_debug_hint(void *addr)
{ … }

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS

static const struct debug_obj_descr i915_sw_fence_debug_descr = …;

static inline void debug_fence_init(struct i915_sw_fence *fence)
{ … }

static inline __maybe_unused void debug_fence_init_onstack(struct i915_sw_fence *fence)
{ … }

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{ … }

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{ … }

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{ … }

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{ … }

static inline __maybe_unused void debug_fence_free(struct i915_sw_fence *fence)
{ … }

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{ … }

#else

static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

static inline __maybe_unused void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

static inline __maybe_unused void debug_fence_free(struct i915_sw_fence *fence)
{
}

static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

#endif

static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{ … }

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
void i915_sw_fence_fini(struct i915_sw_fence *fence)
{ … }
#endif

static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{ … }

static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{ … }

void i915_sw_fence_complete(struct i915_sw_fence *fence)
{ … }

bool i915_sw_fence_await(struct i915_sw_fence *fence)
{ … }

void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{ … }

void i915_sw_fence_reinit(struct i915_sw_fence *fence)
{ … }

void i915_sw_fence_commit(struct i915_sw_fence *fence)
{ … }

static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{ … }

#ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{ … }

static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{ … }

static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{ … }
#else
static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
					 const struct i915_sw_fence * const signaler)
{
	return false;
}
#endif

static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{ … }

int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{ … }

int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{ … }

struct i915_sw_dma_fence_cb_timer { … };

static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{ … }

static void timer_i915_sw_fence_wake(struct timer_list *t)
{ … }

static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
					 struct dma_fence_cb *data)
{ … }

static void irq_i915_sw_fence_work(struct irq_work *wrk)
{ … }

int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{ … }

static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
				     struct dma_fence_cb *data)
{ … }

int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				    struct dma_fence *dma,
				    struct i915_sw_dma_fence_cb *cb)
{ … }

int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct dma_resv *resv,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{ … }

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/lib_sw_fence.c"
#include "selftests/i915_sw_fence.c"
#endif