// SPDX-License-Identifier: MIT
/*
* Copyright © 2014 Intel Corporation
*/
#include <linux/circ_buf.h>
#include "gem/i915_gem_context.h"
#include "gem/i915_gem_lmem.h"
#include "gt/gen8_engine_cs.h"
#include "gt/intel_breadcrumbs.h"
#include "gt/intel_context.h"
#include "gt/intel_engine_heartbeat.h"
#include "gt/intel_engine_pm.h"
#include "gt/intel_engine_regs.h"
#include "gt/intel_gpu_commands.h"
#include "gt/intel_gt.h"
#include "gt/intel_gt_clock_utils.h"
#include "gt/intel_gt_irq.h"
#include "gt/intel_gt_pm.h"
#include "gt/intel_gt_regs.h"
#include "gt/intel_gt_requests.h"
#include "gt/intel_lrc.h"
#include "gt/intel_lrc_reg.h"
#include "gt/intel_mocs.h"
#include "gt/intel_ring.h"
#include "intel_guc_ads.h"
#include "intel_guc_capture.h"
#include "intel_guc_print.h"
#include "intel_guc_submission.h"
#include "i915_drv.h"
#include "i915_reg.h"
#include "i915_irq.h"
#include "i915_trace.h"
/**
* DOC: GuC-based command submission
*
* The Scratch registers:
* There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
* a value to the action register (SOFT_SCRATCH_0) along with any data. It then
* triggers an interrupt on the GuC via another register write (0xC4C8).
* Firmware writes a success/fail code back to the action register after
* processes the request. The kernel driver polls waiting for this update and
* then proceeds.
*
* Command Transport buffers (CTBs):
* Covered in detail in other sections but CTBs (Host to GuC - H2G, GuC to Host
* - G2H) are a message interface between the i915 and GuC.
*
* Context registration:
* Before a context can be submitted it must be registered with the GuC via a
* H2G. A unique guc_id is associated with each context. The context is either
* registered at request creation time (normal operation) or at submission time
* (abnormal operation, e.g. after a reset).
*
* Context submission:
* The i915 updates the LRC tail value in memory. The i915 must enable the
* scheduling of the context within the GuC for the GuC to actually consider it.
* Therefore, the first time a disabled context is submitted we use a schedule
* enable H2G, while follow up submissions are done via the context submit H2G,
* which informs the GuC that a previously enabled context has new work
* available.
*
* Context unpin:
* To unpin a context a H2G is used to disable scheduling. When the
* corresponding G2H returns indicating the scheduling disable operation has
* completed it is safe to unpin the context. While a disable is in flight it
* isn't safe to resubmit the context so a fence is used to stall all future
* requests of that context until the G2H is returned. Because this interaction
* with the GuC takes a non-zero amount of time we delay the disabling of
* scheduling after the pin count goes to zero by a configurable period of time
* (see SCHED_DISABLE_DELAY_MS). The thought is this gives the user a window of
* time to resubmit something on the context before doing this costly operation.
* This delay is only done if the context isn't closed and the guc_id usage is
* less than a threshold (see NUM_SCHED_DISABLE_GUC_IDS_THRESHOLD).
*
* Context deregistration:
* Before a context can be destroyed or if we steal its guc_id we must
* deregister the context with the GuC via H2G. If stealing the guc_id it isn't
* safe to submit anything to this guc_id until the deregister completes so a
* fence is used to stall all requests associated with this guc_id until the
* corresponding G2H returns indicating the guc_id has been deregistered.
*
* submission_state.guc_ids:
* Unique number associated with private GuC context data passed in during
* context registration / submission / deregistration. 64k available. Simple ida
* is used for allocation.
*
* Stealing guc_ids:
* If no guc_ids are available they can be stolen from another context at
* request creation time if that context is unpinned. If a guc_id can't be found
* we punt this problem to the user as we believe this is near impossible to hit
* during normal use cases.
*
* Locking:
* In the GuC submission code we have 3 basic spin locks which protect
* everything. Details about each below.
*
* sched_engine->lock
* This is the submission lock for all contexts that share an i915 schedule
* engine (sched_engine), thus only one of the contexts which share a
* sched_engine can be submitting at a time. Currently only one sched_engine is
* used for all of GuC submission but that could change in the future.
*
* guc->submission_state.lock
* Global lock for GuC submission state. Protects guc_ids and destroyed contexts
* list.
*
* ce->guc_state.lock
* Protects everything under ce->guc_state. Ensures that a context is in the
* correct state before issuing a H2G. e.g. We don't issue a schedule disable
* on a disabled context (bad idea), we don't issue a schedule enable when a
* schedule disable is in flight, etc... Also protects list of inflight requests
* on the context and the priority management state. Lock is individual to each
* context.
*
* Lock ordering rules:
* sched_engine->lock -> ce->guc_state.lock
* guc->submission_state.lock -> ce->guc_state.lock
*
* Reset races:
* When a full GT reset is triggered it is assumed that some G2H responses to
* H2Gs can be lost as the GuC is also reset. Losing these G2H can prove to be
* fatal as we do certain operations upon receiving a G2H (e.g. destroy
* contexts, release guc_ids, etc...). When this occurs we can scrub the
* context state and cleanup appropriately, however this is quite racey.
* To avoid races, the reset code must disable submission before scrubbing for
* the missing G2H, while the submission code must check for submission being
* disabled and skip sending H2Gs and updating context states when it is. Both
* sides must also make sure to hold the relevant locks.
*/
/* GuC Virtual Engine */
struct guc_virtual_engine {
struct intel_engine_cs base;
struct intel_context context;
};
static struct intel_context *
guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
unsigned long flags);
static struct intel_context *
guc_create_parallel(struct intel_engine_cs **engines,
unsigned int num_siblings,
unsigned int width);
#define GUC_REQUEST_SIZE 64 /* bytes */
/*
* We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous
* per the GuC submission interface. A different allocation algorithm is used
* (bitmap vs. ida) between multi-lrc and single-lrc hence the reason to
* partition the guc_id space. We believe the number of multi-lrc contexts in
* use should be low and 1/16 should be sufficient. Minimum of 32 guc_ids for
* multi-lrc.
*/
#define NUMBER_MULTI_LRC_GUC_ID(guc) \
((guc)->submission_state.num_guc_ids / 16)
/*
* Below is a set of functions which control the GuC scheduling state which
* require a lock.
*/
#define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER BIT(0)
#define SCHED_STATE_DESTROYED BIT(1)
#define SCHED_STATE_PENDING_DISABLE BIT(2)
#define SCHED_STATE_BANNED BIT(3)
#define SCHED_STATE_ENABLED BIT(4)
#define SCHED_STATE_PENDING_ENABLE BIT(5)
#define SCHED_STATE_REGISTERED BIT(6)
#define SCHED_STATE_POLICY_REQUIRED BIT(7)
#define SCHED_STATE_CLOSED BIT(8)
#define SCHED_STATE_BLOCKED_SHIFT 9
#define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT)
#define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT)
static inline void init_sched_state(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK;
}
/*
* Kernel contexts can have SCHED_STATE_REGISTERED after suspend.
* A context close can race with the submission path, so SCHED_STATE_CLOSED
* can be set immediately before we try to register.
*/
#define SCHED_STATE_VALID_INIT \
(SCHED_STATE_BLOCKED_MASK | \
SCHED_STATE_CLOSED | \
SCHED_STATE_REGISTERED)
__maybe_unused
static bool sched_state_is_init(struct intel_context *ce)
{
return !(ce->guc_state.sched_state & ~SCHED_STATE_VALID_INIT);
}
static inline bool
context_wait_for_deregister_to_register(struct intel_context *ce)
{
return ce->guc_state.sched_state &
SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
}
static inline void
set_context_wait_for_deregister_to_register(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |=
SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
}
static inline void
clr_context_wait_for_deregister_to_register(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &=
~SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
}
static inline bool
context_destroyed(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_DESTROYED;
}
static inline void
set_context_destroyed(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_DESTROYED;
}
static inline void
clr_context_destroyed(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_DESTROYED;
}
static inline bool context_pending_disable(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_PENDING_DISABLE;
}
static inline void set_context_pending_disable(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_PENDING_DISABLE;
}
static inline void clr_context_pending_disable(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_DISABLE;
}
static inline bool context_banned(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_BANNED;
}
static inline void set_context_banned(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_BANNED;
}
static inline void clr_context_banned(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_BANNED;
}
static inline bool context_enabled(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_ENABLED;
}
static inline void set_context_enabled(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_ENABLED;
}
static inline void clr_context_enabled(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_ENABLED;
}
static inline bool context_pending_enable(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_PENDING_ENABLE;
}
static inline void set_context_pending_enable(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_PENDING_ENABLE;
}
static inline void clr_context_pending_enable(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_ENABLE;
}
static inline bool context_registered(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_REGISTERED;
}
static inline void set_context_registered(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_REGISTERED;
}
static inline void clr_context_registered(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED;
}
static inline bool context_policy_required(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_POLICY_REQUIRED;
}
static inline void set_context_policy_required(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_POLICY_REQUIRED;
}
static inline void clr_context_policy_required(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state &= ~SCHED_STATE_POLICY_REQUIRED;
}
static inline bool context_close_done(struct intel_context *ce)
{
return ce->guc_state.sched_state & SCHED_STATE_CLOSED;
}
static inline void set_context_close_done(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state |= SCHED_STATE_CLOSED;
}
static inline u32 context_blocked(struct intel_context *ce)
{
return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >>
SCHED_STATE_BLOCKED_SHIFT;
}
static inline void incr_context_blocked(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
ce->guc_state.sched_state += SCHED_STATE_BLOCKED;
GEM_BUG_ON(!context_blocked(ce)); /* Overflow check */
}
static inline void decr_context_blocked(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
GEM_BUG_ON(!context_blocked(ce)); /* Underflow check */
ce->guc_state.sched_state -= SCHED_STATE_BLOCKED;
}
static struct intel_context *
request_to_scheduling_context(struct i915_request *rq)
{
return intel_context_to_parent(rq->context);
}
static inline bool context_guc_id_invalid(struct intel_context *ce)
{
return ce->guc_id.id == GUC_INVALID_CONTEXT_ID;
}
static inline void set_context_guc_id_invalid(struct intel_context *ce)
{
ce->guc_id.id = GUC_INVALID_CONTEXT_ID;
}
static inline struct intel_guc *ce_to_guc(struct intel_context *ce)
{
return gt_to_guc(ce->engine->gt);
}
static inline struct i915_priolist *to_priolist(struct rb_node *rb)
{
return rb_entry(rb, struct i915_priolist, node);
}
/*
* When using multi-lrc submission a scratch memory area is reserved in the
* parent's context state for the process descriptor, work queue, and handshake
* between the parent + children contexts to insert safe preemption points
* between each of the BBs. Currently the scratch area is sized to a page.
*
* The layout of this scratch area is below:
* 0 guc_process_desc
* + sizeof(struct guc_process_desc) child go
* + CACHELINE_BYTES child join[0]
* ...
* + CACHELINE_BYTES child join[n - 1]
* ... unused
* PARENT_SCRATCH_SIZE / 2 work queue start
* ... work queue
* PARENT_SCRATCH_SIZE - 1 work queue end
*/
#define WQ_SIZE (PARENT_SCRATCH_SIZE / 2)
#define WQ_OFFSET (PARENT_SCRATCH_SIZE - WQ_SIZE)
struct sync_semaphore {
u32 semaphore;
u8 unused[CACHELINE_BYTES - sizeof(u32)];
};
struct parent_scratch {
union guc_descs {
struct guc_sched_wq_desc wq_desc;
struct guc_process_desc_v69 pdesc;
} descs;
struct sync_semaphore go;
struct sync_semaphore join[MAX_ENGINE_INSTANCE + 1];
u8 unused[WQ_OFFSET - sizeof(union guc_descs) -
sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE + 2)];
u32 wq[WQ_SIZE / sizeof(u32)];
};
static u32 __get_parent_scratch_offset(struct intel_context *ce)
{
GEM_BUG_ON(!ce->parallel.guc.parent_page);
return ce->parallel.guc.parent_page * PAGE_SIZE;
}
static u32 __get_wq_offset(struct intel_context *ce)
{
BUILD_BUG_ON(offsetof(struct parent_scratch, wq) != WQ_OFFSET);
return __get_parent_scratch_offset(ce) + WQ_OFFSET;
}
static struct parent_scratch *
__get_parent_scratch(struct intel_context *ce)
{
BUILD_BUG_ON(sizeof(struct parent_scratch) != PARENT_SCRATCH_SIZE);
BUILD_BUG_ON(sizeof(struct sync_semaphore) != CACHELINE_BYTES);
/*
* Need to subtract LRC_STATE_OFFSET here as the
* parallel.guc.parent_page is the offset into ce->state while
* ce->lrc_reg_reg is ce->state + LRC_STATE_OFFSET.
*/
return (struct parent_scratch *)
(ce->lrc_reg_state +
((__get_parent_scratch_offset(ce) -
LRC_STATE_OFFSET) / sizeof(u32)));
}
static struct guc_process_desc_v69 *
__get_process_desc_v69(struct intel_context *ce)
{
struct parent_scratch *ps = __get_parent_scratch(ce);
return &ps->descs.pdesc;
}
static struct guc_sched_wq_desc *
__get_wq_desc_v70(struct intel_context *ce)
{
struct parent_scratch *ps = __get_parent_scratch(ce);
return &ps->descs.wq_desc;
}
static u32 *get_wq_pointer(struct intel_context *ce, u32 wqi_size)
{
/*
* Check for space in work queue. Caching a value of head pointer in
* intel_context structure in order reduce the number accesses to shared
* GPU memory which may be across a PCIe bus.
*/
#define AVAILABLE_SPACE \
CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE)
if (wqi_size > AVAILABLE_SPACE) {
ce->parallel.guc.wqi_head = READ_ONCE(*ce->parallel.guc.wq_head);
if (wqi_size > AVAILABLE_SPACE)
return NULL;
}
#undef AVAILABLE_SPACE
return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)];
}
static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id)
{
struct intel_context *ce = xa_load(&guc->context_lookup, id);
GEM_BUG_ON(id >= GUC_MAX_CONTEXT_ID);
return ce;
}
static struct guc_lrc_desc_v69 *__get_lrc_desc_v69(struct intel_guc *guc, u32 index)
{
struct guc_lrc_desc_v69 *base = guc->lrc_desc_pool_vaddr_v69;
if (!base)
return NULL;
GEM_BUG_ON(index >= GUC_MAX_CONTEXT_ID);
return &base[index];
}
static int guc_lrc_desc_pool_create_v69(struct intel_guc *guc)
{
u32 size;
int ret;
size = PAGE_ALIGN(sizeof(struct guc_lrc_desc_v69) *
GUC_MAX_CONTEXT_ID);
ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool_v69,
(void **)&guc->lrc_desc_pool_vaddr_v69);
if (ret)
return ret;
return 0;
}
static void guc_lrc_desc_pool_destroy_v69(struct intel_guc *guc)
{
if (!guc->lrc_desc_pool_vaddr_v69)
return;
guc->lrc_desc_pool_vaddr_v69 = NULL;
i915_vma_unpin_and_release(&guc->lrc_desc_pool_v69, I915_VMA_RELEASE_MAP);
}
static inline bool guc_submission_initialized(struct intel_guc *guc)
{
return guc->submission_initialized;
}
static inline void _reset_lrc_desc_v69(struct intel_guc *guc, u32 id)
{
struct guc_lrc_desc_v69 *desc = __get_lrc_desc_v69(guc, id);
if (desc)
memset(desc, 0, sizeof(*desc));
}
static inline bool ctx_id_mapped(struct intel_guc *guc, u32 id)
{
return __get_context(guc, id);
}
static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id,
struct intel_context *ce)
{
unsigned long flags;
/*
* xarray API doesn't have xa_save_irqsave wrapper, so calling the
* lower level functions directly.
*/
xa_lock_irqsave(&guc->context_lookup, flags);
__xa_store(&guc->context_lookup, id, ce, GFP_ATOMIC);
xa_unlock_irqrestore(&guc->context_lookup, flags);
}
static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id)
{
unsigned long flags;
if (unlikely(!guc_submission_initialized(guc)))
return;
_reset_lrc_desc_v69(guc, id);
/*
* xarray API doesn't have xa_erase_irqsave wrapper, so calling
* the lower level functions directly.
*/
xa_lock_irqsave(&guc->context_lookup, flags);
__xa_erase(&guc->context_lookup, id);
xa_unlock_irqrestore(&guc->context_lookup, flags);
}
static void decr_outstanding_submission_g2h(struct intel_guc *guc)
{
if (atomic_dec_and_test(&guc->outstanding_submission_g2h))
wake_up_all(&guc->ct.wq);
}
static int guc_submission_send_busy_loop(struct intel_guc *guc,
const u32 *action,
u32 len,
u32 g2h_len_dw,
bool loop)
{
int ret;
/*
* We always loop when a send requires a reply (i.e. g2h_len_dw > 0),
* so we don't handle the case where we don't get a reply because we
* aborted the send due to the channel being busy.
*/
GEM_BUG_ON(g2h_len_dw && !loop);
if (g2h_len_dw)
atomic_inc(&guc->outstanding_submission_g2h);
ret = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
if (ret)
atomic_dec(&guc->outstanding_submission_g2h);
return ret;
}
int intel_guc_wait_for_pending_msg(struct intel_guc *guc,
atomic_t *wait_var,
bool interruptible,
long timeout)
{
const int state = interruptible ?
TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
DEFINE_WAIT(wait);
might_sleep();
GEM_BUG_ON(timeout < 0);
if (!atomic_read(wait_var))
return 0;
if (!timeout)
return -ETIME;
for (;;) {
prepare_to_wait(&guc->ct.wq, &wait, state);
if (!atomic_read(wait_var))
break;
if (signal_pending_state(state, current)) {
timeout = -EINTR;
break;
}
if (!timeout) {
timeout = -ETIME;
break;
}
timeout = io_schedule_timeout(timeout);
}
finish_wait(&guc->ct.wq, &wait);
return (timeout < 0) ? timeout : 0;
}
int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout)
{
if (!intel_uc_uses_guc_submission(&guc_to_gt(guc)->uc))
return 0;
return intel_guc_wait_for_pending_msg(guc,
&guc->outstanding_submission_g2h,
true, timeout);
}
static int guc_context_policy_init_v70(struct intel_context *ce, bool loop);
static int try_context_registration(struct intel_context *ce, bool loop);
static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)
{
int err = 0;
struct intel_context *ce = request_to_scheduling_context(rq);
u32 action[3];
int len = 0;
u32 g2h_len_dw = 0;
bool enabled;
lockdep_assert_held(&rq->engine->sched_engine->lock);
/*
* Corner case where requests were sitting in the priority list or a
* request resubmitted after the context was banned.
*/
if (unlikely(!intel_context_is_schedulable(ce))) {
i915_request_put(i915_request_mark_eio(rq));
intel_engine_signal_breadcrumbs(ce->engine);
return 0;
}
GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));
GEM_BUG_ON(context_guc_id_invalid(ce));
if (context_policy_required(ce)) {
err = guc_context_policy_init_v70(ce, false);
if (err)
return err;
}
spin_lock(&ce->guc_state.lock);
/*
* The request / context will be run on the hardware when scheduling
* gets enabled in the unblock. For multi-lrc we still submit the
* context to move the LRC tails.
*/
if (unlikely(context_blocked(ce) && !intel_context_is_parent(ce)))
goto out;
enabled = context_enabled(ce) || context_blocked(ce);
if (!enabled) {
action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
action[len++] = ce->guc_id.id;
action[len++] = GUC_CONTEXT_ENABLE;
set_context_pending_enable(ce);
intel_context_get(ce);
g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;
} else {
action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT;
action[len++] = ce->guc_id.id;
}
err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
if (!enabled && !err) {
trace_intel_context_sched_enable(ce);
atomic_inc(&guc->outstanding_submission_g2h);
set_context_enabled(ce);
/*
* Without multi-lrc KMD does the submission step (moving the
* lrc tail) so enabling scheduling is sufficient to submit the
* context. This isn't the case in multi-lrc submission as the
* GuC needs to move the tails, hence the need for another H2G
* to submit a multi-lrc context after enabling scheduling.
*/
if (intel_context_is_parent(ce)) {
action[0] = INTEL_GUC_ACTION_SCHED_CONTEXT;
err = intel_guc_send_nb(guc, action, len - 1, 0);
}
} else if (!enabled) {
clr_context_pending_enable(ce);
intel_context_put(ce);
}
if (likely(!err))
trace_i915_request_guc_submit(rq);
out:
spin_unlock(&ce->guc_state.lock);
return err;
}
static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)
{
int ret = __guc_add_request(guc, rq);
if (unlikely(ret == -EBUSY)) {
guc->stalled_request = rq;
guc->submission_stall_reason = STALL_ADD_REQUEST;
}
return ret;
}
static inline void guc_set_lrc_tail(struct i915_request *rq)
{
rq->context->lrc_reg_state[CTX_RING_TAIL] =
intel_ring_set_tail(rq->ring, rq->tail);
}
static inline int rq_prio(const struct i915_request *rq)
{
return rq->sched.attr.priority;
}
static bool is_multi_lrc_rq(struct i915_request *rq)
{
return intel_context_is_parallel(rq->context);
}
static bool can_merge_rq(struct i915_request *rq,
struct i915_request *last)
{
return request_to_scheduling_context(rq) ==
request_to_scheduling_context(last);
}
static u32 wq_space_until_wrap(struct intel_context *ce)
{
return (WQ_SIZE - ce->parallel.guc.wqi_tail);
}
static void write_wqi(struct intel_context *ce, u32 wqi_size)
{
BUILD_BUG_ON(!is_power_of_2(WQ_SIZE));
/*
* Ensure WQI are visible before updating tail
*/
intel_guc_write_barrier(ce_to_guc(ce));
ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) &
(WQ_SIZE - 1);
WRITE_ONCE(*ce->parallel.guc.wq_tail, ce->parallel.guc.wqi_tail);
}
static int guc_wq_noop_append(struct intel_context *ce)
{
u32 *wqi = get_wq_pointer(ce, wq_space_until_wrap(ce));
u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1;
if (!wqi)
return -EBUSY;
GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));
*wqi = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
FIELD_PREP(WQ_LEN_MASK, len_dw);
ce->parallel.guc.wqi_tail = 0;
return 0;
}
static int __guc_wq_item_append(struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
struct intel_context *child;
unsigned int wqi_size = (ce->parallel.number_children + 4) *
sizeof(u32);
u32 *wqi;
u32 len_dw = (wqi_size / sizeof(u32)) - 1;
int ret;
/* Ensure context is in correct state updating work queue */
GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));
GEM_BUG_ON(context_guc_id_invalid(ce));
GEM_BUG_ON(context_wait_for_deregister_to_register(ce));
GEM_BUG_ON(!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id));
/* Insert NOOP if this work queue item will wrap the tail pointer. */
if (wqi_size > wq_space_until_wrap(ce)) {
ret = guc_wq_noop_append(ce);
if (ret)
return ret;
}
wqi = get_wq_pointer(ce, wqi_size);
if (!wqi)
return -EBUSY;
GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));
*wqi++ = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |
FIELD_PREP(WQ_LEN_MASK, len_dw);
*wqi++ = ce->lrc.lrca;
*wqi++ = FIELD_PREP(WQ_GUC_ID_MASK, ce->guc_id.id) |
FIELD_PREP(WQ_RING_TAIL_MASK, ce->ring->tail / sizeof(u64));
*wqi++ = 0; /* fence_id */
for_each_child(ce, child)
*wqi++ = child->ring->tail / sizeof(u64);
write_wqi(ce, wqi_size);
return 0;
}
static int guc_wq_item_append(struct intel_guc *guc,
struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
int ret;
if (unlikely(!intel_context_is_schedulable(ce)))
return 0;
ret = __guc_wq_item_append(rq);
if (unlikely(ret == -EBUSY)) {
guc->stalled_request = rq;
guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
}
return ret;
}
static bool multi_lrc_submit(struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
intel_ring_set_tail(rq->ring, rq->tail);
/*
* We expect the front end (execbuf IOCTL) to set this flag on the last
* request generated from a multi-BB submission. This indicates to the
* backend (GuC interface) that we should submit this context thus
* submitting all the requests generated in parallel.
*/
return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) ||
!intel_context_is_schedulable(ce);
}
static int guc_dequeue_one_context(struct intel_guc *guc)
{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
struct i915_request *last = NULL;
bool submit = false;
struct rb_node *rb;
int ret;
lockdep_assert_held(&sched_engine->lock);
if (guc->stalled_request) {
submit = true;
last = guc->stalled_request;
switch (guc->submission_stall_reason) {
case STALL_REGISTER_CONTEXT:
goto register_context;
case STALL_MOVE_LRC_TAIL:
goto move_lrc_tail;
case STALL_ADD_REQUEST:
goto add_request;
default:
MISSING_CASE(guc->submission_stall_reason);
}
}
while ((rb = rb_first_cached(&sched_engine->queue))) {
struct i915_priolist *p = to_priolist(rb);
struct i915_request *rq, *rn;
priolist_for_each_request_consume(rq, rn, p) {
if (last && !can_merge_rq(rq, last))
goto register_context;
list_del_init(&rq->sched.link);
__i915_request_submit(rq);
trace_i915_request_in(rq, 0);
last = rq;
if (is_multi_lrc_rq(rq)) {
/*
* We need to coalesce all multi-lrc requests in
* a relationship into a single H2G. We are
* guaranteed that all of these requests will be
* submitted sequentially.
*/
if (multi_lrc_submit(rq)) {
submit = true;
goto register_context;
}
} else {
submit = true;
}
}
rb_erase_cached(&p->node, &sched_engine->queue);
i915_priolist_free(p);
}
register_context:
if (submit) {
struct intel_context *ce = request_to_scheduling_context(last);
if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&
intel_context_is_schedulable(ce))) {
ret = try_context_registration(ce, false);
if (unlikely(ret == -EPIPE)) {
goto deadlk;
} else if (ret == -EBUSY) {
guc->stalled_request = last;
guc->submission_stall_reason =
STALL_REGISTER_CONTEXT;
goto schedule_tasklet;
} else if (ret != 0) {
GEM_WARN_ON(ret); /* Unexpected */
goto deadlk;
}
}
move_lrc_tail:
if (is_multi_lrc_rq(last)) {
ret = guc_wq_item_append(guc, last);
if (ret == -EBUSY) {
goto schedule_tasklet;
} else if (ret != 0) {
GEM_WARN_ON(ret); /* Unexpected */
goto deadlk;
}
} else {
guc_set_lrc_tail(last);
}
add_request:
ret = guc_add_request(guc, last);
if (unlikely(ret == -EPIPE)) {
goto deadlk;
} else if (ret == -EBUSY) {
goto schedule_tasklet;
} else if (ret != 0) {
GEM_WARN_ON(ret); /* Unexpected */
goto deadlk;
}
}
guc->stalled_request = NULL;
guc->submission_stall_reason = STALL_NONE;
return submit;
deadlk:
sched_engine->tasklet.callback = NULL;
tasklet_disable_nosync(&sched_engine->tasklet);
return false;
schedule_tasklet:
tasklet_schedule(&sched_engine->tasklet);
return false;
}
static void guc_submission_tasklet(struct tasklet_struct *t)
{
struct i915_sched_engine *sched_engine =
from_tasklet(sched_engine, t, tasklet);
unsigned long flags;
bool loop;
spin_lock_irqsave(&sched_engine->lock, flags);
do {
loop = guc_dequeue_one_context(sched_engine->private_data);
} while (loop);
i915_sched_engine_reset_on_empty(sched_engine);
spin_unlock_irqrestore(&sched_engine->lock, flags);
}
static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir)
{
if (iir & GT_RENDER_USER_INTERRUPT)
intel_engine_signal_breadcrumbs(engine);
}
static void __guc_context_destroy(struct intel_context *ce);
static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);
static void guc_signal_context_fence(struct intel_context *ce);
static void guc_cancel_context_requests(struct intel_context *ce);
static void guc_blocked_fence_complete(struct intel_context *ce);
static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
{
struct intel_context *ce;
unsigned long index, flags;
bool pending_disable, pending_enable, deregister, destroyed, banned;
xa_lock_irqsave(&guc->context_lookup, flags);
xa_for_each(&guc->context_lookup, index, ce) {
/*
* Corner case where the ref count on the object is zero but and
* deregister G2H was lost. In this case we don't touch the ref
* count and finish the destroy of the context.
*/
bool do_put = kref_get_unless_zero(&ce->ref);
xa_unlock(&guc->context_lookup);
if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&
(cancel_delayed_work(&ce->guc_state.sched_disable_delay_work))) {
/* successful cancel so jump straight to close it */
intel_context_sched_disable_unpin(ce);
}
spin_lock(&ce->guc_state.lock);
/*
* Once we are at this point submission_disabled() is guaranteed
* to be visible to all callers who set the below flags (see above
* flush and flushes in reset_prepare). If submission_disabled()
* is set, the caller shouldn't set these flags.
*/
destroyed = context_destroyed(ce);
pending_enable = context_pending_enable(ce);
pending_disable = context_pending_disable(ce);
deregister = context_wait_for_deregister_to_register(ce);
banned = context_banned(ce);
init_sched_state(ce);
spin_unlock(&ce->guc_state.lock);
if (pending_enable || destroyed || deregister) {
decr_outstanding_submission_g2h(guc);
if (deregister)
guc_signal_context_fence(ce);
if (destroyed) {
intel_gt_pm_put_async_untracked(guc_to_gt(guc));
release_guc_id(guc, ce);
__guc_context_destroy(ce);
}
if (pending_enable || deregister)
intel_context_put(ce);
}
/* Not mutualy exclusive with above if statement. */
if (pending_disable) {
guc_signal_context_fence(ce);
if (banned) {
guc_cancel_context_requests(ce);
intel_engine_signal_breadcrumbs(ce->engine);
}
intel_context_sched_disable_unpin(ce);
decr_outstanding_submission_g2h(guc);
spin_lock(&ce->guc_state.lock);
guc_blocked_fence_complete(ce);
spin_unlock(&ce->guc_state.lock);
intel_context_put(ce);
}
if (do_put)
intel_context_put(ce);
xa_lock(&guc->context_lookup);
}
xa_unlock_irqrestore(&guc->context_lookup, flags);
}
/*
* GuC stores busyness stats for each engine at context in/out boundaries. A
* context 'in' logs execution start time, 'out' adds in -> out delta to total.
* i915/kmd accesses 'start', 'total' and 'context id' from memory shared with
* GuC.
*
* __i915_pmu_event_read samples engine busyness. When sampling, if context id
* is valid (!= ~0) and start is non-zero, the engine is considered to be
* active. For an active engine total busyness = total + (now - start), where
* 'now' is the time at which the busyness is sampled. For inactive engine,
* total busyness = total.
*
* All times are captured from GUCPMTIMESTAMP reg and are in gt clock domain.
*
* The start and total values provided by GuC are 32 bits and wrap around in a
* few minutes. Since perf pmu provides busyness as 64 bit monotonically
* increasing ns values, there is a need for this implementation to account for
* overflows and extend the GuC provided values to 64 bits before returning
* busyness to the user. In order to do that, a worker runs periodically at
* frequency = 1/8th the time it takes for the timestamp to wrap (i.e. once in
* 27 seconds for a gt clock frequency of 19.2 MHz).
*/
#define WRAP_TIME_CLKS U32_MAX
#define POLL_TIME_CLKS (WRAP_TIME_CLKS >> 3)
static void
__extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start)
{
u32 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
u32 gt_stamp_last = lower_32_bits(guc->timestamp.gt_stamp);
if (new_start == lower_32_bits(*prev_start))
return;
/*
* When gt is unparked, we update the gt timestamp and start the ping
* worker that updates the gt_stamp every POLL_TIME_CLKS. As long as gt
* is unparked, all switched in contexts will have a start time that is
* within +/- POLL_TIME_CLKS of the most recent gt_stamp.
*
* If neither gt_stamp nor new_start has rolled over, then the
* gt_stamp_hi does not need to be adjusted, however if one of them has
* rolled over, we need to adjust gt_stamp_hi accordingly.
*
* The below conditions address the cases of new_start rollover and
* gt_stamp_last rollover respectively.
*/
if (new_start < gt_stamp_last &&
(new_start - gt_stamp_last) <= POLL_TIME_CLKS)
gt_stamp_hi++;
if (new_start > gt_stamp_last &&
(gt_stamp_last - new_start) <= POLL_TIME_CLKS && gt_stamp_hi)
gt_stamp_hi--;
*prev_start = ((u64)gt_stamp_hi << 32) | new_start;
}
#define record_read(map_, field_) \
iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_)
/*
* GuC updates shared memory and KMD reads it. Since this is not synchronized,
* we run into a race where the value read is inconsistent. Sometimes the
* inconsistency is in reading the upper MSB bytes of the last_in value when
* this race occurs. 2 types of cases are seen - upper 8 bits are zero and upper
* 24 bits are zero. Since these are non-zero values, it is non-trivial to
* determine validity of these values. Instead we read the values multiple times
* until they are consistent. In test runs, 3 attempts results in consistent
* values. The upper bound is set to 6 attempts and may need to be tuned as per
* any new occurences.
*/
static void __get_engine_usage_record(struct intel_engine_cs *engine,
u32 *last_in, u32 *id, u32 *total)
{
struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine);
int i = 0;
do {
*last_in = record_read(&rec_map, last_switch_in_stamp);
*id = record_read(&rec_map, current_context_index);
*total = record_read(&rec_map, total_runtime);
if (record_read(&rec_map, last_switch_in_stamp) == *last_in &&
record_read(&rec_map, current_context_index) == *id &&
record_read(&rec_map, total_runtime) == *total)
break;
} while (++i < 6);
}
static void guc_update_engine_gt_clks(struct intel_engine_cs *engine)
{
struct intel_engine_guc_stats *stats = &engine->stats.guc;
struct intel_guc *guc = gt_to_guc(engine->gt);
u32 last_switch, ctx_id, total;
lockdep_assert_held(&guc->timestamp.lock);
__get_engine_usage_record(engine, &last_switch, &ctx_id, &total);
stats->running = ctx_id != ~0U && last_switch;
if (stats->running)
__extend_last_switch(guc, &stats->start_gt_clk, last_switch);
/*
* Instead of adjusting the total for overflow, just add the
* difference from previous sample stats->total_gt_clks
*/
if (total && total != ~0U) {
stats->total_gt_clks += (u32)(total - stats->prev_total);
stats->prev_total = total;
}
}
static u32 gpm_timestamp_shift(struct intel_gt *gt)
{
intel_wakeref_t wakeref;
u32 reg, shift;
with_intel_runtime_pm(gt->uncore->rpm, wakeref)
reg = intel_uncore_read(gt->uncore, RPM_CONFIG0);
shift = (reg & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>
GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT;
return 3 - shift;
}
static void guc_update_pm_timestamp(struct intel_guc *guc, ktime_t *now)
{
struct intel_gt *gt = guc_to_gt(guc);
u32 gt_stamp_lo, gt_stamp_hi;
u64 gpm_ts;
lockdep_assert_held(&guc->timestamp.lock);
gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
gpm_ts = intel_uncore_read64_2x32(gt->uncore, MISC_STATUS0,
MISC_STATUS1) >> guc->timestamp.shift;
gt_stamp_lo = lower_32_bits(gpm_ts);
*now = ktime_get();
if (gt_stamp_lo < lower_32_bits(guc->timestamp.gt_stamp))
gt_stamp_hi++;
guc->timestamp.gt_stamp = ((u64)gt_stamp_hi << 32) | gt_stamp_lo;
}
/*
* Unlike the execlist mode of submission total and active times are in terms of
* gt clocks. The *now parameter is retained to return the cpu time at which the
* busyness was sampled.
*/
static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)
{
struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;
struct i915_gpu_error *gpu_error = &engine->i915->gpu_error;
struct intel_gt *gt = engine->gt;
struct intel_guc *guc = gt_to_guc(gt);
u64 total, gt_stamp_saved;
unsigned long flags;
u32 reset_count;
bool in_reset;
intel_wakeref_t wakeref;
spin_lock_irqsave(&guc->timestamp.lock, flags);
/*
* If a reset happened, we risk reading partially updated engine
* busyness from GuC, so we just use the driver stored copy of busyness.
* Synchronize with gt reset using reset_count and the
* I915_RESET_BACKOFF flag. Note that reset flow updates the reset_count
* after I915_RESET_BACKOFF flag, so ensure that the reset_count is
* usable by checking the flag afterwards.
*/
reset_count = i915_reset_count(gpu_error);
in_reset = test_bit(I915_RESET_BACKOFF, >->reset.flags);
*now = ktime_get();
/*
* The active busyness depends on start_gt_clk and gt_stamp.
* gt_stamp is updated by i915 only when gt is awake and the
* start_gt_clk is derived from GuC state. To get a consistent
* view of activity, we query the GuC state only if gt is awake.
*/
wakeref = in_reset ? 0 : intel_gt_pm_get_if_awake(gt);
if (wakeref) {
stats_saved = *stats;
gt_stamp_saved = guc->timestamp.gt_stamp;
/*
* Update gt_clks, then gt timestamp to simplify the 'gt_stamp -
* start_gt_clk' calculation below for active engines.
*/
guc_update_engine_gt_clks(engine);
guc_update_pm_timestamp(guc, now);
intel_gt_pm_put_async(gt, wakeref);
if (i915_reset_count(gpu_error) != reset_count) {
*stats = stats_saved;
guc->timestamp.gt_stamp = gt_stamp_saved;
}
}
total = intel_gt_clock_interval_to_ns(gt, stats->total_gt_clks);
if (stats->running) {
u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;
total += intel_gt_clock_interval_to_ns(gt, clk);
}
spin_unlock_irqrestore(&guc->timestamp.lock, flags);
return ns_to_ktime(total);
}
static void guc_enable_busyness_worker(struct intel_guc *guc)
{
mod_delayed_work(system_highpri_wq, &guc->timestamp.work, guc->timestamp.ping_delay);
}
static void guc_cancel_busyness_worker(struct intel_guc *guc)
{
/*
* There are many different call stacks that can get here. Some of them
* hold the reset mutex. The busyness worker also attempts to acquire the
* reset mutex. Synchronously flushing a worker thread requires acquiring
* the worker mutex. Lockdep sees this as a conflict. It thinks that the
* flush can deadlock because it holds the worker mutex while waiting for
* the reset mutex, but another thread is holding the reset mutex and might
* attempt to use other worker functions.
*
* In practice, this scenario does not exist because the busyness worker
* does not block waiting for the reset mutex. It does a try-lock on it and
* immediately exits if the lock is already held. Unfortunately, the mutex
* in question (I915_RESET_BACKOFF) is an i915 implementation which has lockdep
* annotation but not to the extent of explaining the 'might lock' is also a
* 'does not need to lock'. So one option would be to add more complex lockdep
* annotations to ignore the issue (if at all possible). A simpler option is to
* just not flush synchronously when a rest in progress. Given that the worker
* will just early exit and re-schedule itself anyway, there is no advantage
* to running it immediately.
*
* If a reset is not in progress, then the synchronous flush may be required.
* As noted many call stacks lead here, some during suspend and driver unload
* which do require a synchronous flush to make sure the worker is stopped
* before memory is freed.
*
* Trying to pass a 'need_sync' or 'in_reset' flag all the way down through
* every possible call stack is unfeasible. It would be too intrusive to many
* areas that really don't care about the GuC backend. However, there is the
* I915_RESET_BACKOFF flag and the gt->reset.mutex can be tested for is_locked.
* So just use those. Note that testing both is required due to the hideously
* complex nature of the i915 driver's reset code paths.
*
* And note that in the case of a reset occurring during driver unload
* (wedged_on_fini), skipping the cancel in reset_prepare/reset_fini (when the
* reset flag/mutex are set) is fine because there is another explicit cancel in
* intel_guc_submission_fini (when the reset flag/mutex are not).
*/
if (mutex_is_locked(&guc_to_gt(guc)->reset.mutex) ||
test_bit(I915_RESET_BACKOFF, &guc_to_gt(guc)->reset.flags))
cancel_delayed_work(&guc->timestamp.work);
else
cancel_delayed_work_sync(&guc->timestamp.work);
}
static void __reset_guc_busyness_stats(struct intel_guc *guc)
{
struct intel_gt *gt = guc_to_gt(guc);
struct intel_engine_cs *engine;
enum intel_engine_id id;
unsigned long flags;
ktime_t unused;
spin_lock_irqsave(&guc->timestamp.lock, flags);
guc_update_pm_timestamp(guc, &unused);
for_each_engine(engine, gt, id) {
guc_update_engine_gt_clks(engine);
engine->stats.guc.prev_total = 0;
}
spin_unlock_irqrestore(&guc->timestamp.lock, flags);
}
static void __update_guc_busyness_stats(struct intel_guc *guc)
{
struct intel_gt *gt = guc_to_gt(guc);
struct intel_engine_cs *engine;
enum intel_engine_id id;
unsigned long flags;
ktime_t unused;
guc->timestamp.last_stat_jiffies = jiffies;
spin_lock_irqsave(&guc->timestamp.lock, flags);
guc_update_pm_timestamp(guc, &unused);
for_each_engine(engine, gt, id)
guc_update_engine_gt_clks(engine);
spin_unlock_irqrestore(&guc->timestamp.lock, flags);
}
static void __guc_context_update_stats(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
spin_lock_irqsave(&guc->timestamp.lock, flags);
lrc_update_runtime(ce);
spin_unlock_irqrestore(&guc->timestamp.lock, flags);
}
static void guc_context_update_stats(struct intel_context *ce)
{
if (!intel_context_pin_if_active(ce))
return;
__guc_context_update_stats(ce);
intel_context_unpin(ce);
}
static void guc_timestamp_ping(struct work_struct *wrk)
{
struct intel_guc *guc = container_of(wrk, typeof(*guc),
timestamp.work.work);
struct intel_uc *uc = container_of(guc, typeof(*uc), guc);
struct intel_gt *gt = guc_to_gt(guc);
struct intel_context *ce;
intel_wakeref_t wakeref;
unsigned long index;
int srcu, ret;
/*
* Ideally the busyness worker should take a gt pm wakeref because the
* worker only needs to be active while gt is awake. However, the
* gt_park path cancels the worker synchronously and this complicates
* the flow if the worker is also running at the same time. The cancel
* waits for the worker and when the worker releases the wakeref, that
* would call gt_park and would lead to a deadlock.
*
* The resolution is to take the global pm wakeref if runtime pm is
* already active. If not, we don't need to update the busyness stats as
* the stats would already be updated when the gt was parked.
*
* Note:
* - We do not requeue the worker if we cannot take a reference to runtime
* pm since intel_guc_busyness_unpark would requeue the worker in the
* resume path.
*
* - If the gt was parked longer than time taken for GT timestamp to roll
* over, we ignore those rollovers since we don't care about tracking
* the exact GT time. We only care about roll overs when the gt is
* active and running workloads.
*
* - There is a window of time between gt_park and runtime suspend,
* where the worker may run. This is acceptable since the worker will
* not find any new data to update busyness.
*/
wakeref = intel_runtime_pm_get_if_active(>->i915->runtime_pm);
if (!wakeref)
return;
/*
* Synchronize with gt reset to make sure the worker does not
* corrupt the engine/guc stats. NB: can't actually block waiting
* for a reset to complete as the reset requires flushing out
* this worker thread if started. So waiting would deadlock.
*/
ret = intel_gt_reset_trylock(gt, &srcu);
if (ret)
goto err_trylock;
__update_guc_busyness_stats(guc);
/* adjust context stats for overflow */
xa_for_each(&guc->context_lookup, index, ce)
guc_context_update_stats(ce);
intel_gt_reset_unlock(gt, srcu);
guc_enable_busyness_worker(guc);
err_trylock:
intel_runtime_pm_put(>->i915->runtime_pm, wakeref);
}
static int guc_action_enable_usage_stats(struct intel_guc *guc)
{
u32 offset = intel_guc_engine_usage_offset(guc);
u32 action[] = {
INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF,
offset,
0,
};
return intel_guc_send(guc, action, ARRAY_SIZE(action));
}
static int guc_init_engine_stats(struct intel_guc *guc)
{
struct intel_gt *gt = guc_to_gt(guc);
intel_wakeref_t wakeref;
int ret;
with_intel_runtime_pm(>->i915->runtime_pm, wakeref)
ret = guc_action_enable_usage_stats(guc);
if (ret)
guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));
else
guc_enable_busyness_worker(guc);
return ret;
}
static void guc_fini_engine_stats(struct intel_guc *guc)
{
guc_cancel_busyness_worker(guc);
}
void intel_guc_busyness_park(struct intel_gt *gt)
{
struct intel_guc *guc = gt_to_guc(gt);
if (!guc_submission_initialized(guc))
return;
/*
* There is a race with suspend flow where the worker runs after suspend
* and causes an unclaimed register access warning. Cancel the worker
* synchronously here.
*/
guc_cancel_busyness_worker(guc);
/*
* Before parking, we should sample engine busyness stats if we need to.
* We can skip it if we are less than half a ping from the last time we
* sampled the busyness stats.
*/
if (guc->timestamp.last_stat_jiffies &&
!time_after(jiffies, guc->timestamp.last_stat_jiffies +
(guc->timestamp.ping_delay / 2)))
return;
__update_guc_busyness_stats(guc);
}
void intel_guc_busyness_unpark(struct intel_gt *gt)
{
struct intel_guc *guc = gt_to_guc(gt);
unsigned long flags;
ktime_t unused;
if (!guc_submission_initialized(guc))
return;
spin_lock_irqsave(&guc->timestamp.lock, flags);
guc_update_pm_timestamp(guc, &unused);
spin_unlock_irqrestore(&guc->timestamp.lock, flags);
guc_enable_busyness_worker(guc);
}
static inline bool
submission_disabled(struct intel_guc *guc)
{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
return unlikely(!sched_engine ||
!__tasklet_is_enabled(&sched_engine->tasklet) ||
intel_gt_is_wedged(guc_to_gt(guc)));
}
static void disable_submission(struct intel_guc *guc)
{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
if (__tasklet_is_enabled(&sched_engine->tasklet)) {
GEM_BUG_ON(!guc->ct.enabled);
__tasklet_disable_sync_once(&sched_engine->tasklet);
sched_engine->tasklet.callback = NULL;
}
}
static void enable_submission(struct intel_guc *guc)
{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
unsigned long flags;
spin_lock_irqsave(&guc->sched_engine->lock, flags);
sched_engine->tasklet.callback = guc_submission_tasklet;
wmb(); /* Make sure callback visible */
if (!__tasklet_is_enabled(&sched_engine->tasklet) &&
__tasklet_enable(&sched_engine->tasklet)) {
GEM_BUG_ON(!guc->ct.enabled);
/* And kick in case we missed a new request submission. */
tasklet_hi_schedule(&sched_engine->tasklet);
}
spin_unlock_irqrestore(&guc->sched_engine->lock, flags);
}
static void guc_flush_submissions(struct intel_guc *guc)
{
struct i915_sched_engine * const sched_engine = guc->sched_engine;
unsigned long flags;
spin_lock_irqsave(&sched_engine->lock, flags);
spin_unlock_irqrestore(&sched_engine->lock, flags);
}
void intel_guc_submission_flush_work(struct intel_guc *guc)
{
flush_work(&guc->submission_state.destroyed_worker);
}
static void guc_flush_destroyed_contexts(struct intel_guc *guc);
void intel_guc_submission_reset_prepare(struct intel_guc *guc)
{
if (unlikely(!guc_submission_initialized(guc))) {
/* Reset called during driver load? GuC not yet initialised! */
return;
}
intel_gt_park_heartbeats(guc_to_gt(guc));
disable_submission(guc);
guc->interrupts.disable(guc);
__reset_guc_busyness_stats(guc);
/* Flush IRQ handler */
spin_lock_irq(guc_to_gt(guc)->irq_lock);
spin_unlock_irq(guc_to_gt(guc)->irq_lock);
guc_flush_submissions(guc);
guc_flush_destroyed_contexts(guc);
flush_work(&guc->ct.requests.worker);
scrub_guc_desc_for_outstanding_g2h(guc);
}
static struct intel_engine_cs *
guc_virtual_get_sibling(struct intel_engine_cs *ve, unsigned int sibling)
{
struct intel_engine_cs *engine;
intel_engine_mask_t tmp, mask = ve->mask;
unsigned int num_siblings = 0;
for_each_engine_masked(engine, ve->gt, mask, tmp)
if (num_siblings++ == sibling)
return engine;
return NULL;
}
static inline struct intel_engine_cs *
__context_to_physical_engine(struct intel_context *ce)
{
struct intel_engine_cs *engine = ce->engine;
if (intel_engine_is_virtual(engine))
engine = guc_virtual_get_sibling(engine, 0);
return engine;
}
static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub)
{
struct intel_engine_cs *engine = __context_to_physical_engine(ce);
if (!intel_context_is_schedulable(ce))
return;
GEM_BUG_ON(!intel_context_is_pinned(ce));
/*
* We want a simple context + ring to execute the breadcrumb update.
* We cannot rely on the context being intact across the GPU hang,
* so clear it and rebuild just what we need for the breadcrumb.
* All pending requests for this context will be zapped, and any
* future request will be after userspace has had the opportunity
* to recreate its own state.
*/
if (scrub)
lrc_init_regs(ce, engine, true);
/* Rerun the request; its payload has been neutered (if guilty). */
lrc_update_regs(ce, engine, head);
}
static void guc_engine_reset_prepare(struct intel_engine_cs *engine)
{
/*
* Wa_22011802037: In addition to stopping the cs, we need
* to wait for any pending mi force wakeups
*/
if (intel_engine_reset_needs_wa_22011802037(engine->gt)) {
intel_engine_stop_cs(engine);
intel_engine_wait_for_pending_mi_fw(engine);
}
}
static void guc_reset_nop(struct intel_engine_cs *engine)
{
}
static void guc_rewind_nop(struct intel_engine_cs *engine, bool stalled)
{
}
static void
__unwind_incomplete_requests(struct intel_context *ce)
{
struct i915_request *rq, *rn;
struct list_head *pl;
int prio = I915_PRIORITY_INVALID;
struct i915_sched_engine * const sched_engine =
ce->engine->sched_engine;
unsigned long flags;
spin_lock_irqsave(&sched_engine->lock, flags);
spin_lock(&ce->guc_state.lock);
list_for_each_entry_safe_reverse(rq, rn,
&ce->guc_state.requests,
sched.link) {
if (i915_request_completed(rq))
continue;
list_del_init(&rq->sched.link);
__i915_request_unsubmit(rq);
/* Push the request back into the queue for later resubmission. */
GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
if (rq_prio(rq) != prio) {
prio = rq_prio(rq);
pl = i915_sched_lookup_priolist(sched_engine, prio);
}
GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine));
list_add(&rq->sched.link, pl);
set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
}
spin_unlock(&ce->guc_state.lock);
spin_unlock_irqrestore(&sched_engine->lock, flags);
}
static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t stalled)
{
bool guilty;
struct i915_request *rq;
unsigned long flags;
u32 head;
int i, number_children = ce->parallel.number_children;
struct intel_context *parent = ce;
GEM_BUG_ON(intel_context_is_child(ce));
intel_context_get(ce);
/*
* GuC will implicitly mark the context as non-schedulable when it sends
* the reset notification. Make sure our state reflects this change. The
* context will be marked enabled on resubmission.
*/
spin_lock_irqsave(&ce->guc_state.lock, flags);
clr_context_enabled(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
/*
* For each context in the relationship find the hanging request
* resetting each context / request as needed
*/
for (i = 0; i < number_children + 1; ++i) {
if (!intel_context_is_pinned(ce))
goto next_context;
guilty = false;
rq = intel_context_get_active_request(ce);
if (!rq) {
head = ce->ring->tail;
goto out_replay;
}
if (i915_request_started(rq))
guilty = stalled & ce->engine->mask;
GEM_BUG_ON(i915_active_is_idle(&ce->active));
head = intel_ring_wrap(ce->ring, rq->head);
__i915_request_reset(rq, guilty);
i915_request_put(rq);
out_replay:
guc_reset_state(ce, head, guilty);
next_context:
if (i != number_children)
ce = list_next_entry(ce, parallel.child_link);
}
__unwind_incomplete_requests(parent);
intel_context_put(parent);
}
void wake_up_all_tlb_invalidate(struct intel_guc *guc)
{
struct intel_guc_tlb_wait *wait;
unsigned long i;
if (!intel_guc_tlb_invalidation_is_available(guc))
return;
xa_lock_irq(&guc->tlb_lookup);
xa_for_each(&guc->tlb_lookup, i, wait)
wake_up(&wait->wq);
xa_unlock_irq(&guc->tlb_lookup);
}
void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
{
struct intel_context *ce;
unsigned long index;
unsigned long flags;
if (unlikely(!guc_submission_initialized(guc))) {
/* Reset called during driver load? GuC not yet initialised! */
return;
}
xa_lock_irqsave(&guc->context_lookup, flags);
xa_for_each(&guc->context_lookup, index, ce) {
if (!kref_get_unless_zero(&ce->ref))
continue;
xa_unlock(&guc->context_lookup);
if (intel_context_is_pinned(ce) &&
!intel_context_is_child(ce))
__guc_reset_context(ce, stalled);
intel_context_put(ce);
xa_lock(&guc->context_lookup);
}
xa_unlock_irqrestore(&guc->context_lookup, flags);
/* GuC is blown away, drop all references to contexts */
xa_destroy(&guc->context_lookup);
}
static void guc_cancel_context_requests(struct intel_context *ce)
{
struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine;
struct i915_request *rq;
unsigned long flags;
/* Mark all executing requests as skipped. */
spin_lock_irqsave(&sched_engine->lock, flags);
spin_lock(&ce->guc_state.lock);
list_for_each_entry(rq, &ce->guc_state.requests, sched.link)
i915_request_put(i915_request_mark_eio(rq));
spin_unlock(&ce->guc_state.lock);
spin_unlock_irqrestore(&sched_engine->lock, flags);
}
static void
guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine)
{
struct i915_request *rq, *rn;
struct rb_node *rb;
unsigned long flags;
/* Can be called during boot if GuC fails to load */
if (!sched_engine)
return;
/*
* Before we call engine->cancel_requests(), we should have exclusive
* access to the submission state. This is arranged for us by the
* caller disabling the interrupt generation, the tasklet and other
* threads that may then access the same state, giving us a free hand
* to reset state. However, we still need to let lockdep be aware that
* we know this state may be accessed in hardirq context, so we
* disable the irq around this manipulation and we want to keep
* the spinlock focused on its duties and not accidentally conflate
* coverage to the submission's irq state. (Similarly, although we
* shouldn't need to disable irq around the manipulation of the
* submission's irq state, we also wish to remind ourselves that
* it is irq state.)
*/
spin_lock_irqsave(&sched_engine->lock, flags);
/* Flush the queued requests to the timeline list (for retiring). */
while ((rb = rb_first_cached(&sched_engine->queue))) {
struct i915_priolist *p = to_priolist(rb);
priolist_for_each_request_consume(rq, rn, p) {
list_del_init(&rq->sched.link);
__i915_request_submit(rq);
i915_request_put(i915_request_mark_eio(rq));
}
rb_erase_cached(&p->node, &sched_engine->queue);
i915_priolist_free(p);
}
/* Remaining _unready_ requests will be nop'ed when submitted */
sched_engine->queue_priority_hint = INT_MIN;
sched_engine->queue = RB_ROOT_CACHED;
spin_unlock_irqrestore(&sched_engine->lock, flags);
}
void intel_guc_submission_cancel_requests(struct intel_guc *guc)
{
struct intel_context *ce;
unsigned long index;
unsigned long flags;
xa_lock_irqsave(&guc->context_lookup, flags);
xa_for_each(&guc->context_lookup, index, ce) {
if (!kref_get_unless_zero(&ce->ref))
continue;
xa_unlock(&guc->context_lookup);
if (intel_context_is_pinned(ce) &&
!intel_context_is_child(ce))
guc_cancel_context_requests(ce);
intel_context_put(ce);
xa_lock(&guc->context_lookup);
}
xa_unlock_irqrestore(&guc->context_lookup, flags);
guc_cancel_sched_engine_requests(guc->sched_engine);
/* GuC is blown away, drop all references to contexts */
xa_destroy(&guc->context_lookup);
/*
* Wedged GT won't respond to any TLB invalidation request. Simply
* release all the blocked waiters.
*/
wake_up_all_tlb_invalidate(guc);
}
void intel_guc_submission_reset_finish(struct intel_guc *guc)
{
/* Reset called during driver load or during wedge? */
if (unlikely(!guc_submission_initialized(guc) ||
!intel_guc_is_fw_running(guc) ||
intel_gt_is_wedged(guc_to_gt(guc)))) {
return;
}
/*
* Technically possible for either of these values to be non-zero here,
* but very unlikely + harmless. Regardless let's add an error so we can
* see in CI if this happens frequently / a precursor to taking down the
* machine.
*/
if (atomic_read(&guc->outstanding_submission_g2h))
guc_err(guc, "Unexpected outstanding GuC to Host in reset finish\n");
atomic_set(&guc->outstanding_submission_g2h, 0);
intel_guc_global_policies_update(guc);
enable_submission(guc);
intel_gt_unpark_heartbeats(guc_to_gt(guc));
/*
* The full GT reset will have cleared the TLB caches and flushed the
* G2H message queue; we can release all the blocked waiters.
*/
wake_up_all_tlb_invalidate(guc);
}
static void destroyed_worker_func(struct work_struct *w);
static void reset_fail_worker_func(struct work_struct *w);
bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc)
{
return HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915) &&
intel_guc_is_ready(guc);
}
static int init_tlb_lookup(struct intel_guc *guc)
{
struct intel_guc_tlb_wait *wait;
int err;
if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
return 0;
xa_init_flags(&guc->tlb_lookup, XA_FLAGS_ALLOC);
wait = kzalloc(sizeof(*wait), GFP_KERNEL);
if (!wait)
return -ENOMEM;
init_waitqueue_head(&wait->wq);
/* Preallocate a shared id for use under memory pressure. */
err = xa_alloc_cyclic_irq(&guc->tlb_lookup, &guc->serial_slot, wait,
xa_limit_32b, &guc->next_seqno, GFP_KERNEL);
if (err < 0) {
kfree(wait);
return err;
}
return 0;
}
static void fini_tlb_lookup(struct intel_guc *guc)
{
struct intel_guc_tlb_wait *wait;
if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
return;
wait = xa_load(&guc->tlb_lookup, guc->serial_slot);
if (wait && wait->busy)
guc_err(guc, "Unexpected busy item in tlb_lookup on fini\n");
kfree(wait);
xa_destroy(&guc->tlb_lookup);
}
/*
* Set up the memory resources to be shared with the GuC (via the GGTT)
* at firmware loading time.
*/
int intel_guc_submission_init(struct intel_guc *guc)
{
struct intel_gt *gt = guc_to_gt(guc);
int ret;
if (guc->submission_initialized)
return 0;
if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 0, 0)) {
ret = guc_lrc_desc_pool_create_v69(guc);
if (ret)
return ret;
}
ret = init_tlb_lookup(guc);
if (ret)
goto destroy_pool;
guc->submission_state.guc_ids_bitmap =
bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL);
if (!guc->submission_state.guc_ids_bitmap) {
ret = -ENOMEM;
goto destroy_tlb;
}
guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ;
guc->timestamp.shift = gpm_timestamp_shift(gt);
guc->submission_initialized = true;
return 0;
destroy_tlb:
fini_tlb_lookup(guc);
destroy_pool:
guc_lrc_desc_pool_destroy_v69(guc);
return ret;
}
void intel_guc_submission_fini(struct intel_guc *guc)
{
if (!guc->submission_initialized)
return;
guc_fini_engine_stats(guc);
guc_flush_destroyed_contexts(guc);
guc_lrc_desc_pool_destroy_v69(guc);
i915_sched_engine_put(guc->sched_engine);
bitmap_free(guc->submission_state.guc_ids_bitmap);
fini_tlb_lookup(guc);
guc->submission_initialized = false;
}
static inline void queue_request(struct i915_sched_engine *sched_engine,
struct i915_request *rq,
int prio)
{
GEM_BUG_ON(!list_empty(&rq->sched.link));
list_add_tail(&rq->sched.link,
i915_sched_lookup_priolist(sched_engine, prio));
set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
tasklet_hi_schedule(&sched_engine->tasklet);
}
static int guc_bypass_tasklet_submit(struct intel_guc *guc,
struct i915_request *rq)
{
int ret = 0;
__i915_request_submit(rq);
trace_i915_request_in(rq, 0);
if (is_multi_lrc_rq(rq)) {
if (multi_lrc_submit(rq)) {
ret = guc_wq_item_append(guc, rq);
if (!ret)
ret = guc_add_request(guc, rq);
}
} else {
guc_set_lrc_tail(rq);
ret = guc_add_request(guc, rq);
}
if (unlikely(ret == -EPIPE))
disable_submission(guc);
return ret;
}
static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)
{
struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
struct intel_context *ce = request_to_scheduling_context(rq);
return submission_disabled(guc) || guc->stalled_request ||
!i915_sched_engine_is_empty(sched_engine) ||
!ctx_id_mapped(guc, ce->guc_id.id);
}
static void guc_submit_request(struct i915_request *rq)
{
struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
struct intel_guc *guc = gt_to_guc(rq->engine->gt);
unsigned long flags;
/* Will be called from irq-context when using foreign fences. */
spin_lock_irqsave(&sched_engine->lock, flags);
if (need_tasklet(guc, rq))
queue_request(sched_engine, rq, rq_prio(rq));
else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY)
tasklet_hi_schedule(&sched_engine->tasklet);
spin_unlock_irqrestore(&sched_engine->lock, flags);
}
static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
int ret;
GEM_BUG_ON(intel_context_is_child(ce));
if (intel_context_is_parent(ce))
ret = bitmap_find_free_region(guc->submission_state.guc_ids_bitmap,
NUMBER_MULTI_LRC_GUC_ID(guc),
order_base_2(ce->parallel.number_children
+ 1));
else
ret = ida_alloc_range(&guc->submission_state.guc_ids,
NUMBER_MULTI_LRC_GUC_ID(guc),
guc->submission_state.num_guc_ids - 1,
GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
if (unlikely(ret < 0))
return ret;
if (!intel_context_is_parent(ce))
++guc->submission_state.guc_ids_in_use;
ce->guc_id.id = ret;
return 0;
}
static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
GEM_BUG_ON(intel_context_is_child(ce));
if (!context_guc_id_invalid(ce)) {
if (intel_context_is_parent(ce)) {
bitmap_release_region(guc->submission_state.guc_ids_bitmap,
ce->guc_id.id,
order_base_2(ce->parallel.number_children
+ 1));
} else {
--guc->submission_state.guc_ids_in_use;
ida_free(&guc->submission_state.guc_ids,
ce->guc_id.id);
}
clr_ctx_id_mapping(guc, ce->guc_id.id);
set_context_guc_id_invalid(ce);
}
if (!list_empty(&ce->guc_id.link))
list_del_init(&ce->guc_id.link);
}
static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
unsigned long flags;
spin_lock_irqsave(&guc->submission_state.lock, flags);
__release_guc_id(guc, ce);
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
}
static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
struct intel_context *cn;
lockdep_assert_held(&guc->submission_state.lock);
GEM_BUG_ON(intel_context_is_child(ce));
GEM_BUG_ON(intel_context_is_parent(ce));
if (!list_empty(&guc->submission_state.guc_id_list)) {
cn = list_first_entry(&guc->submission_state.guc_id_list,
struct intel_context,
guc_id.link);
GEM_BUG_ON(atomic_read(&cn->guc_id.ref));
GEM_BUG_ON(context_guc_id_invalid(cn));
GEM_BUG_ON(intel_context_is_child(cn));
GEM_BUG_ON(intel_context_is_parent(cn));
list_del_init(&cn->guc_id.link);
ce->guc_id.id = cn->guc_id.id;
spin_lock(&cn->guc_state.lock);
clr_context_registered(cn);
spin_unlock(&cn->guc_state.lock);
set_context_guc_id_invalid(cn);
#ifdef CONFIG_DRM_I915_SELFTEST
guc->number_guc_id_stolen++;
#endif
return 0;
} else {
return -EAGAIN;
}
}
static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
int ret;
lockdep_assert_held(&guc->submission_state.lock);
GEM_BUG_ON(intel_context_is_child(ce));
ret = new_guc_id(guc, ce);
if (unlikely(ret < 0)) {
if (intel_context_is_parent(ce))
return -ENOSPC;
ret = steal_guc_id(guc, ce);
if (ret < 0)
return ret;
}
if (intel_context_is_parent(ce)) {
struct intel_context *child;
int i = 1;
for_each_child(ce, child)
child->guc_id.id = ce->guc_id.id + i++;
}
return 0;
}
#define PIN_GUC_ID_TRIES 4
static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
int ret = 0;
unsigned long flags, tries = PIN_GUC_ID_TRIES;
GEM_BUG_ON(atomic_read(&ce->guc_id.ref));
try_again:
spin_lock_irqsave(&guc->submission_state.lock, flags);
might_lock(&ce->guc_state.lock);
if (context_guc_id_invalid(ce)) {
ret = assign_guc_id(guc, ce);
if (ret)
goto out_unlock;
ret = 1; /* Indidcates newly assigned guc_id */
}
if (!list_empty(&ce->guc_id.link))
list_del_init(&ce->guc_id.link);
atomic_inc(&ce->guc_id.ref);
out_unlock:
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
/*
* -EAGAIN indicates no guc_id are available, let's retire any
* outstanding requests to see if that frees up a guc_id. If the first
* retire didn't help, insert a sleep with the timeslice duration before
* attempting to retire more requests. Double the sleep period each
* subsequent pass before finally giving up. The sleep period has max of
* 100ms and minimum of 1ms.
*/
if (ret == -EAGAIN && --tries) {
if (PIN_GUC_ID_TRIES - tries > 1) {
unsigned int timeslice_shifted =
ce->engine->props.timeslice_duration_ms <<
(PIN_GUC_ID_TRIES - tries - 2);
unsigned int max = min_t(unsigned int, 100,
timeslice_shifted);
msleep(max_t(unsigned int, max, 1));
}
intel_gt_retire_requests(guc_to_gt(guc));
goto try_again;
}
return ret;
}
static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce)
{
unsigned long flags;
GEM_BUG_ON(atomic_read(&ce->guc_id.ref) < 0);
GEM_BUG_ON(intel_context_is_child(ce));
if (unlikely(context_guc_id_invalid(ce) ||
intel_context_is_parent(ce)))
return;
spin_lock_irqsave(&guc->submission_state.lock, flags);
if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id.link) &&
!atomic_read(&ce->guc_id.ref))
list_add_tail(&ce->guc_id.link,
&guc->submission_state.guc_id_list);
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
}
static int __guc_action_register_multi_lrc_v69(struct intel_guc *guc,
struct intel_context *ce,
u32 guc_id,
u32 offset,
bool loop)
{
struct intel_context *child;
u32 action[4 + MAX_ENGINE_INSTANCE];
int len = 0;
GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);
action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
action[len++] = guc_id;
action[len++] = ce->parallel.number_children + 1;
action[len++] = offset;
for_each_child(ce, child) {
offset += sizeof(struct guc_lrc_desc_v69);
action[len++] = offset;
}
return guc_submission_send_busy_loop(guc, action, len, 0, loop);
}
static int __guc_action_register_multi_lrc_v70(struct intel_guc *guc,
struct intel_context *ce,
struct guc_ctxt_registration_info *info,
bool loop)
{
struct intel_context *child;
u32 action[13 + (MAX_ENGINE_INSTANCE * 2)];
int len = 0;
u32 next_id;
GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);
action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
action[len++] = info->flags;
action[len++] = info->context_idx;
action[len++] = info->engine_class;
action[len++] = info->engine_submit_mask;
action[len++] = info->wq_desc_lo;
action[len++] = info->wq_desc_hi;
action[len++] = info->wq_base_lo;
action[len++] = info->wq_base_hi;
action[len++] = info->wq_size;
action[len++] = ce->parallel.number_children + 1;
action[len++] = info->hwlrca_lo;
action[len++] = info->hwlrca_hi;
next_id = info->context_idx + 1;
for_each_child(ce, child) {
GEM_BUG_ON(next_id++ != child->guc_id.id);
/*
* NB: GuC interface supports 64 bit LRCA even though i915/HW
* only supports 32 bit currently.
*/
action[len++] = lower_32_bits(child->lrc.lrca);
action[len++] = upper_32_bits(child->lrc.lrca);
}
GEM_BUG_ON(len > ARRAY_SIZE(action));
return guc_submission_send_busy_loop(guc, action, len, 0, loop);
}
static int __guc_action_register_context_v69(struct intel_guc *guc,
u32 guc_id,
u32 offset,
bool loop)
{
u32 action[] = {
INTEL_GUC_ACTION_REGISTER_CONTEXT,
guc_id,
offset,
};
return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
0, loop);
}
static int __guc_action_register_context_v70(struct intel_guc *guc,
struct guc_ctxt_registration_info *info,
bool loop)
{
u32 action[] = {
INTEL_GUC_ACTION_REGISTER_CONTEXT,
info->flags,
info->context_idx,
info->engine_class,
info->engine_submit_mask,
info->wq_desc_lo,
info->wq_desc_hi,
info->wq_base_lo,
info->wq_base_hi,
info->wq_size,
info->hwlrca_lo,
info->hwlrca_hi,
};
return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
0, loop);
}
static void prepare_context_registration_info_v69(struct intel_context *ce);
static void prepare_context_registration_info_v70(struct intel_context *ce,
struct guc_ctxt_registration_info *info);
static int
register_context_v69(struct intel_guc *guc, struct intel_context *ce, bool loop)
{
u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool_v69) +
ce->guc_id.id * sizeof(struct guc_lrc_desc_v69);
prepare_context_registration_info_v69(ce);
if (intel_context_is_parent(ce))
return __guc_action_register_multi_lrc_v69(guc, ce, ce->guc_id.id,
offset, loop);
else
return __guc_action_register_context_v69(guc, ce->guc_id.id,
offset, loop);
}
static int
register_context_v70(struct intel_guc *guc, struct intel_context *ce, bool loop)
{
struct guc_ctxt_registration_info info;
prepare_context_registration_info_v70(ce, &info);
if (intel_context_is_parent(ce))
return __guc_action_register_multi_lrc_v70(guc, ce, &info, loop);
else
return __guc_action_register_context_v70(guc, &info, loop);
}
static int register_context(struct intel_context *ce, bool loop)
{
struct intel_guc *guc = ce_to_guc(ce);
int ret;
GEM_BUG_ON(intel_context_is_child(ce));
trace_intel_context_register(ce);
if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
ret = register_context_v70(guc, ce, loop);
else
ret = register_context_v69(guc, ce, loop);
if (likely(!ret)) {
unsigned long flags;
spin_lock_irqsave(&ce->guc_state.lock, flags);
set_context_registered(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
guc_context_policy_init_v70(ce, loop);
}
return ret;
}
static int __guc_action_deregister_context(struct intel_guc *guc,
u32 guc_id)
{
u32 action[] = {
INTEL_GUC_ACTION_DEREGISTER_CONTEXT,
guc_id,
};
return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
G2H_LEN_DW_DEREGISTER_CONTEXT,
true);
}
static int deregister_context(struct intel_context *ce, u32 guc_id)
{
struct intel_guc *guc = ce_to_guc(ce);
GEM_BUG_ON(intel_context_is_child(ce));
trace_intel_context_deregister(ce);
return __guc_action_deregister_context(guc, guc_id);
}
static inline void clear_children_join_go_memory(struct intel_context *ce)
{
struct parent_scratch *ps = __get_parent_scratch(ce);
int i;
ps->go.semaphore = 0;
for (i = 0; i < ce->parallel.number_children + 1; ++i)
ps->join[i].semaphore = 0;
}
static inline u32 get_children_go_value(struct intel_context *ce)
{
return __get_parent_scratch(ce)->go.semaphore;
}
static inline u32 get_children_join_value(struct intel_context *ce,
u8 child_index)
{
return __get_parent_scratch(ce)->join[child_index].semaphore;
}
struct context_policy {
u32 count;
struct guc_update_context_policy h2g;
};
static u32 __guc_context_policy_action_size(struct context_policy *policy)
{
size_t bytes = sizeof(policy->h2g.header) +
(sizeof(policy->h2g.klv[0]) * policy->count);
return bytes / sizeof(u32);
}
static void __guc_context_policy_start_klv(struct context_policy *policy, u16 guc_id)
{
policy->h2g.header.action = INTEL_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
policy->h2g.header.ctx_id = guc_id;
policy->count = 0;
}
#define MAKE_CONTEXT_POLICY_ADD(func, id) \
static void __guc_context_policy_add_##func(struct context_policy *policy, u32 data) \
{ \
GEM_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \
policy->h2g.klv[policy->count].kl = \
FIELD_PREP(GUC_KLV_0_KEY, GUC_CONTEXT_POLICIES_KLV_ID_##id) | \
FIELD_PREP(GUC_KLV_0_LEN, 1); \
policy->h2g.klv[policy->count].value = data; \
policy->count++; \
}
MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY)
MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY)
MAKE_CONTEXT_POLICY_ADD(slpc_ctx_freq_req, SLPM_GT_FREQUENCY)
#undef MAKE_CONTEXT_POLICY_ADD
static int __guc_context_set_context_policies(struct intel_guc *guc,
struct context_policy *policy,
bool loop)
{
return guc_submission_send_busy_loop(guc, (u32 *)&policy->h2g,
__guc_context_policy_action_size(policy),
0, loop);
}
static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
{
struct intel_engine_cs *engine = ce->engine;
struct intel_guc *guc = gt_to_guc(engine->gt);
struct context_policy policy;
u32 execution_quantum;
u32 preemption_timeout;
u32 slpc_ctx_freq_req = 0;
unsigned long flags;
int ret;
/* NB: For both of these, zero means disabled. */
GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,
execution_quantum));
GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,
preemption_timeout));
execution_quantum = engine->props.timeslice_duration_ms * 1000;
preemption_timeout = engine->props.preempt_timeout_ms * 1000;
if (ce->flags & BIT(CONTEXT_LOW_LATENCY))
slpc_ctx_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
__guc_context_policy_start_klv(&policy, ce->guc_id.id);
__guc_context_policy_add_priority(&policy, ce->guc_state.prio);
__guc_context_policy_add_execution_quantum(&policy, execution_quantum);
__guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
__guc_context_policy_add_slpc_ctx_freq_req(&policy, slpc_ctx_freq_req);
if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
__guc_context_policy_add_preempt_to_idle(&policy, 1);
ret = __guc_context_set_context_policies(guc, &policy, loop);
spin_lock_irqsave(&ce->guc_state.lock, flags);
if (ret != 0)
set_context_policy_required(ce);
else
clr_context_policy_required(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
return ret;
}
static void guc_context_policy_init_v69(struct intel_engine_cs *engine,
struct guc_lrc_desc_v69 *desc)
{
desc->policy_flags = 0;
if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE_V69;
/* NB: For both of these, zero means disabled. */
GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,
desc->execution_quantum));
GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,
desc->preemption_timeout));
desc->execution_quantum = engine->props.timeslice_duration_ms * 1000;
desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000;
}
static u32 map_guc_prio_to_lrc_desc_prio(u8 prio)
{
/*
* this matches the mapping we do in map_i915_prio_to_guc_prio()
* (e.g. prio < I915_PRIORITY_NORMAL maps to GUC_CLIENT_PRIORITY_NORMAL)
*/
switch (prio) {
default:
MISSING_CASE(prio);
fallthrough;
case GUC_CLIENT_PRIORITY_KMD_NORMAL:
return GEN12_CTX_PRIORITY_NORMAL;
case GUC_CLIENT_PRIORITY_NORMAL:
return GEN12_CTX_PRIORITY_LOW;
case GUC_CLIENT_PRIORITY_HIGH:
case GUC_CLIENT_PRIORITY_KMD_HIGH:
return GEN12_CTX_PRIORITY_HIGH;
}
}
static void prepare_context_registration_info_v69(struct intel_context *ce)
{
struct intel_engine_cs *engine = ce->engine;
struct intel_guc *guc = gt_to_guc(engine->gt);
u32 ctx_id = ce->guc_id.id;
struct guc_lrc_desc_v69 *desc;
struct intel_context *child;
GEM_BUG_ON(!engine->mask);
/*
* Ensure LRC + CT vmas are is same region as write barrier is done
* based on CT vma region.
*/
GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
i915_gem_object_is_lmem(ce->ring->vma->obj));
desc = __get_lrc_desc_v69(guc, ctx_id);
GEM_BUG_ON(!desc);
desc->engine_class = engine_class_to_guc_class(engine->class);
desc->engine_submit_mask = engine->logical_mask;
desc->hw_context_desc = ce->lrc.lrca;
desc->priority = ce->guc_state.prio;
desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;
guc_context_policy_init_v69(engine, desc);
/*
* If context is a parent, we need to register a process descriptor
* describing a work queue and register all child contexts.
*/
if (intel_context_is_parent(ce)) {
struct guc_process_desc_v69 *pdesc;
ce->parallel.guc.wqi_tail = 0;
ce->parallel.guc.wqi_head = 0;
desc->process_desc = i915_ggtt_offset(ce->state) +
__get_parent_scratch_offset(ce);
desc->wq_addr = i915_ggtt_offset(ce->state) +
__get_wq_offset(ce);
desc->wq_size = WQ_SIZE;
pdesc = __get_process_desc_v69(ce);
memset(pdesc, 0, sizeof(*(pdesc)));
pdesc->stage_id = ce->guc_id.id;
pdesc->wq_base_addr = desc->wq_addr;
pdesc->wq_size_bytes = desc->wq_size;
pdesc->wq_status = WQ_STATUS_ACTIVE;
ce->parallel.guc.wq_head = &pdesc->head;
ce->parallel.guc.wq_tail = &pdesc->tail;
ce->parallel.guc.wq_status = &pdesc->wq_status;
for_each_child(ce, child) {
desc = __get_lrc_desc_v69(guc, child->guc_id.id);
desc->engine_class =
engine_class_to_guc_class(engine->class);
desc->hw_context_desc = child->lrc.lrca;
desc->priority = ce->guc_state.prio;
desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;
guc_context_policy_init_v69(engine, desc);
}
clear_children_join_go_memory(ce);
}
}
static void prepare_context_registration_info_v70(struct intel_context *ce,
struct guc_ctxt_registration_info *info)
{
struct intel_engine_cs *engine = ce->engine;
struct intel_guc *guc = gt_to_guc(engine->gt);
u32 ctx_id = ce->guc_id.id;
GEM_BUG_ON(!engine->mask);
/*
* Ensure LRC + CT vmas are is same region as write barrier is done
* based on CT vma region.
*/
GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
i915_gem_object_is_lmem(ce->ring->vma->obj));
memset(info, 0, sizeof(*info));
info->context_idx = ctx_id;
info->engine_class = engine_class_to_guc_class(engine->class);
info->engine_submit_mask = engine->logical_mask;
/*
* NB: GuC interface supports 64 bit LRCA even though i915/HW
* only supports 32 bit currently.
*/
info->hwlrca_lo = lower_32_bits(ce->lrc.lrca);
info->hwlrca_hi = upper_32_bits(ce->lrc.lrca);
if (engine->flags & I915_ENGINE_HAS_EU_PRIORITY)
info->hwlrca_lo |= map_guc_prio_to_lrc_desc_prio(ce->guc_state.prio);
info->flags = CONTEXT_REGISTRATION_FLAG_KMD;
/*
* If context is a parent, we need to register a process descriptor
* describing a work queue and register all child contexts.
*/
if (intel_context_is_parent(ce)) {
struct guc_sched_wq_desc *wq_desc;
u64 wq_desc_offset, wq_base_offset;
ce->parallel.guc.wqi_tail = 0;
ce->parallel.guc.wqi_head = 0;
wq_desc_offset = (u64)i915_ggtt_offset(ce->state) +
__get_parent_scratch_offset(ce);
wq_base_offset = (u64)i915_ggtt_offset(ce->state) +
__get_wq_offset(ce);
info->wq_desc_lo = lower_32_bits(wq_desc_offset);
info->wq_desc_hi = upper_32_bits(wq_desc_offset);
info->wq_base_lo = lower_32_bits(wq_base_offset);
info->wq_base_hi = upper_32_bits(wq_base_offset);
info->wq_size = WQ_SIZE;
wq_desc = __get_wq_desc_v70(ce);
memset(wq_desc, 0, sizeof(*wq_desc));
wq_desc->wq_status = WQ_STATUS_ACTIVE;
ce->parallel.guc.wq_head = &wq_desc->head;
ce->parallel.guc.wq_tail = &wq_desc->tail;
ce->parallel.guc.wq_status = &wq_desc->wq_status;
clear_children_join_go_memory(ce);
}
}
static int try_context_registration(struct intel_context *ce, bool loop)
{
struct intel_engine_cs *engine = ce->engine;
struct intel_runtime_pm *runtime_pm = engine->uncore->rpm;
struct intel_guc *guc = gt_to_guc(engine->gt);
intel_wakeref_t wakeref;
u32 ctx_id = ce->guc_id.id;
bool context_registered;
int ret = 0;
GEM_BUG_ON(!sched_state_is_init(ce));
context_registered = ctx_id_mapped(guc, ctx_id);
clr_ctx_id_mapping(guc, ctx_id);
set_ctx_id_mapping(guc, ctx_id, ce);
/*
* The context_lookup xarray is used to determine if the hardware
* context is currently registered. There are two cases in which it
* could be registered either the guc_id has been stolen from another
* context or the lrc descriptor address of this context has changed. In
* either case the context needs to be deregistered with the GuC before
* registering this context.
*/
if (context_registered) {
bool disabled;
unsigned long flags;
trace_intel_context_steal_guc_id(ce);
GEM_BUG_ON(!loop);
/* Seal race with Reset */
spin_lock_irqsave(&ce->guc_state.lock, flags);
disabled = submission_disabled(guc);
if (likely(!disabled)) {
set_context_wait_for_deregister_to_register(ce);
intel_context_get(ce);
}
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
if (unlikely(disabled)) {
clr_ctx_id_mapping(guc, ctx_id);
return 0; /* Will get registered later */
}
/*
* If stealing the guc_id, this ce has the same guc_id as the
* context whose guc_id was stolen.
*/
with_intel_runtime_pm(runtime_pm, wakeref)
ret = deregister_context(ce, ce->guc_id.id);
if (unlikely(ret == -ENODEV))
ret = 0; /* Will get registered later */
} else {
with_intel_runtime_pm(runtime_pm, wakeref)
ret = register_context(ce, loop);
if (unlikely(ret == -EBUSY)) {
clr_ctx_id_mapping(guc, ctx_id);
} else if (unlikely(ret == -ENODEV)) {
clr_ctx_id_mapping(guc, ctx_id);
ret = 0; /* Will get registered later */
}
}
return ret;
}
static int __guc_context_pre_pin(struct intel_context *ce,
struct intel_engine_cs *engine,
struct i915_gem_ww_ctx *ww,
void **vaddr)
{
return lrc_pre_pin(ce, engine, ww, vaddr);
}
static int __guc_context_pin(struct intel_context *ce,
struct intel_engine_cs *engine,
void *vaddr)
{
if (i915_ggtt_offset(ce->state) !=
(ce->lrc.lrca & CTX_GTT_ADDRESS_MASK))
set_bit(CONTEXT_LRCA_DIRTY, &ce->flags);
/*
* GuC context gets pinned in guc_request_alloc. See that function for
* explaination of why.
*/
return lrc_pin(ce, engine, vaddr);
}
static int guc_context_pre_pin(struct intel_context *ce,
struct i915_gem_ww_ctx *ww,
void **vaddr)
{
return __guc_context_pre_pin(ce, ce->engine, ww, vaddr);
}
static int guc_context_pin(struct intel_context *ce, void *vaddr)
{
int ret = __guc_context_pin(ce, ce->engine, vaddr);
if (likely(!ret && !intel_context_is_barrier(ce)))
intel_engine_pm_get(ce->engine);
return ret;
}
static void guc_context_unpin(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
__guc_context_update_stats(ce);
unpin_guc_id(guc, ce);
lrc_unpin(ce);
if (likely(!intel_context_is_barrier(ce)))
intel_engine_pm_put_async(ce->engine);
}
static void guc_context_post_unpin(struct intel_context *ce)
{
lrc_post_unpin(ce);
}
static void __guc_context_sched_enable(struct intel_guc *guc,
struct intel_context *ce)
{
u32 action[] = {
INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
ce->guc_id.id,
GUC_CONTEXT_ENABLE
};
trace_intel_context_sched_enable(ce);
guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);
}
static void __guc_context_sched_disable(struct intel_guc *guc,
struct intel_context *ce,
u16 guc_id)
{
u32 action[] = {
INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
guc_id, /* ce->guc_id.id not stable */
GUC_CONTEXT_DISABLE
};
GEM_BUG_ON(guc_id == GUC_INVALID_CONTEXT_ID);
GEM_BUG_ON(intel_context_is_child(ce));
trace_intel_context_sched_disable(ce);
guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);
}
static void guc_blocked_fence_complete(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
if (!i915_sw_fence_done(&ce->guc_state.blocked))
i915_sw_fence_complete(&ce->guc_state.blocked);
}
static void guc_blocked_fence_reinit(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_state.blocked));
/*
* This fence is always complete unless a pending schedule disable is
* outstanding. We arm the fence here and complete it when we receive
* the pending schedule disable complete message.
*/
i915_sw_fence_fini(&ce->guc_state.blocked);
i915_sw_fence_reinit(&ce->guc_state.blocked);
i915_sw_fence_await(&ce->guc_state.blocked);
i915_sw_fence_commit(&ce->guc_state.blocked);
}
static u16 prep_context_pending_disable(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
set_context_pending_disable(ce);
clr_context_enabled(ce);
guc_blocked_fence_reinit(ce);
intel_context_get(ce);
return ce->guc_id.id;
}
static struct i915_sw_fence *guc_context_block(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
intel_wakeref_t wakeref;
u16 guc_id;
bool enabled;
GEM_BUG_ON(intel_context_is_child(ce));
spin_lock_irqsave(&ce->guc_state.lock, flags);
incr_context_blocked(ce);
enabled = context_enabled(ce);
if (unlikely(!enabled || submission_disabled(guc))) {
if (enabled)
clr_context_enabled(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
return &ce->guc_state.blocked;
}
/*
* We add +2 here as the schedule disable complete CTB handler calls
* intel_context_sched_disable_unpin (-2 to pin_count).
*/
atomic_add(2, &ce->pin_count);
guc_id = prep_context_pending_disable(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
with_intel_runtime_pm(runtime_pm, wakeref)
__guc_context_sched_disable(guc, ce, guc_id);
return &ce->guc_state.blocked;
}
#define SCHED_STATE_MULTI_BLOCKED_MASK \
(SCHED_STATE_BLOCKED_MASK & ~SCHED_STATE_BLOCKED)
#define SCHED_STATE_NO_UNBLOCK \
(SCHED_STATE_MULTI_BLOCKED_MASK | \
SCHED_STATE_PENDING_DISABLE | \
SCHED_STATE_BANNED)
static bool context_cant_unblock(struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) ||
context_guc_id_invalid(ce) ||
!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id) ||
!intel_context_is_pinned(ce);
}
static void guc_context_unblock(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
intel_wakeref_t wakeref;
bool enable;
GEM_BUG_ON(context_enabled(ce));
GEM_BUG_ON(intel_context_is_child(ce));
spin_lock_irqsave(&ce->guc_state.lock, flags);
if (unlikely(submission_disabled(guc) ||
context_cant_unblock(ce))) {
enable = false;
} else {
enable = true;
set_context_pending_enable(ce);
set_context_enabled(ce);
intel_context_get(ce);
}
decr_context_blocked(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
if (enable) {
with_intel_runtime_pm(runtime_pm, wakeref)
__guc_context_sched_enable(guc, ce);
}
}
static void guc_context_cancel_request(struct intel_context *ce,
struct i915_request *rq)
{
struct intel_context *block_context =
request_to_scheduling_context(rq);
if (i915_sw_fence_signaled(&rq->submit)) {
struct i915_sw_fence *fence;
intel_context_get(ce);
fence = guc_context_block(block_context);
i915_sw_fence_wait(fence);
if (!i915_request_completed(rq)) {
__i915_request_skip(rq);
guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head),
true);
}
guc_context_unblock(block_context);
intel_context_put(ce);
}
}
static void __guc_context_set_preemption_timeout(struct intel_guc *guc,
u16 guc_id,
u32 preemption_timeout)
{
if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
struct context_policy policy;
__guc_context_policy_start_klv(&policy, guc_id);
__guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
__guc_context_set_context_policies(guc, &policy, true);
} else {
u32 action[] = {
INTEL_GUC_ACTION_V69_SET_CONTEXT_PREEMPTION_TIMEOUT,
guc_id,
preemption_timeout
};
intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
}
}
static void
guc_context_revoke(struct intel_context *ce, struct i915_request *rq,
unsigned int preempt_timeout_ms)
{
struct intel_guc *guc = ce_to_guc(ce);
struct intel_runtime_pm *runtime_pm =
&ce->engine->gt->i915->runtime_pm;
intel_wakeref_t wakeref;
unsigned long flags;
GEM_BUG_ON(intel_context_is_child(ce));
guc_flush_submissions(guc);
spin_lock_irqsave(&ce->guc_state.lock, flags);
set_context_banned(ce);
if (submission_disabled(guc) ||
(!context_enabled(ce) && !context_pending_disable(ce))) {
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
guc_cancel_context_requests(ce);
intel_engine_signal_breadcrumbs(ce->engine);
} else if (!context_pending_disable(ce)) {
u16 guc_id;
/*
* We add +2 here as the schedule disable complete CTB handler
* calls intel_context_sched_disable_unpin (-2 to pin_count).
*/
atomic_add(2, &ce->pin_count);
guc_id = prep_context_pending_disable(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
/*
* In addition to disabling scheduling, set the preemption
* timeout to the minimum value (1 us) so the banned context
* gets kicked off the HW ASAP.
*/
with_intel_runtime_pm(runtime_pm, wakeref) {
__guc_context_set_preemption_timeout(guc, guc_id,
preempt_timeout_ms);
__guc_context_sched_disable(guc, ce, guc_id);
}
} else {
if (!context_guc_id_invalid(ce))
with_intel_runtime_pm(runtime_pm, wakeref)
__guc_context_set_preemption_timeout(guc,
ce->guc_id.id,
preempt_timeout_ms);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
}
}
static void do_sched_disable(struct intel_guc *guc, struct intel_context *ce,
unsigned long flags)
__releases(ce->guc_state.lock)
{
struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;
intel_wakeref_t wakeref;
u16 guc_id;
lockdep_assert_held(&ce->guc_state.lock);
guc_id = prep_context_pending_disable(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
with_intel_runtime_pm(runtime_pm, wakeref)
__guc_context_sched_disable(guc, ce, guc_id);
}
static bool bypass_sched_disable(struct intel_guc *guc,
struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
GEM_BUG_ON(intel_context_is_child(ce));
if (submission_disabled(guc) || context_guc_id_invalid(ce) ||
!ctx_id_mapped(guc, ce->guc_id.id)) {
clr_context_enabled(ce);
return true;
}
return !context_enabled(ce);
}
static void __delay_sched_disable(struct work_struct *wrk)
{
struct intel_context *ce =
container_of(wrk, typeof(*ce), guc_state.sched_disable_delay_work.work);
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
spin_lock_irqsave(&ce->guc_state.lock, flags);
if (bypass_sched_disable(guc, ce)) {
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
intel_context_sched_disable_unpin(ce);
} else {
do_sched_disable(guc, ce, flags);
}
}
static bool guc_id_pressure(struct intel_guc *guc, struct intel_context *ce)
{
/*
* parent contexts are perma-pinned, if we are unpinning do schedule
* disable immediately.
*/
if (intel_context_is_parent(ce))
return true;
/*
* If we are beyond the threshold for avail guc_ids, do schedule disable immediately.
*/
return guc->submission_state.guc_ids_in_use >
guc->submission_state.sched_disable_gucid_threshold;
}
static void guc_context_sched_disable(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
u64 delay = guc->submission_state.sched_disable_delay_ms;
unsigned long flags;
spin_lock_irqsave(&ce->guc_state.lock, flags);
if (bypass_sched_disable(guc, ce)) {
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
intel_context_sched_disable_unpin(ce);
} else if (!intel_context_is_closed(ce) && !guc_id_pressure(guc, ce) &&
delay) {
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
mod_delayed_work(system_unbound_wq,
&ce->guc_state.sched_disable_delay_work,
msecs_to_jiffies(delay));
} else {
do_sched_disable(guc, ce, flags);
}
}
static void guc_context_close(struct intel_context *ce)
{
unsigned long flags;
if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&
cancel_delayed_work(&ce->guc_state.sched_disable_delay_work))
__delay_sched_disable(&ce->guc_state.sched_disable_delay_work.work);
spin_lock_irqsave(&ce->guc_state.lock, flags);
set_context_close_done(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
}
static inline int guc_lrc_desc_unpin(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
struct intel_gt *gt = guc_to_gt(guc);
unsigned long flags;
bool disabled;
int ret;
GEM_BUG_ON(!intel_gt_pm_is_awake(gt));
GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id));
GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id));
GEM_BUG_ON(context_enabled(ce));
/* Seal race with Reset */
spin_lock_irqsave(&ce->guc_state.lock, flags);
disabled = submission_disabled(guc);
if (likely(!disabled)) {
/*
* Take a gt-pm ref and change context state to be destroyed.
* NOTE: a G2H IRQ that comes after will put this gt-pm ref back
*/
__intel_gt_pm_get(gt);
set_context_destroyed(ce);
clr_context_registered(ce);
}
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
if (unlikely(disabled)) {
release_guc_id(guc, ce);
__guc_context_destroy(ce);
return 0;
}
/*
* GuC is active, lets destroy this context, but at this point we can still be racing
* with suspend, so we undo everything if the H2G fails in deregister_context so
* that GuC reset will find this context during clean up.
*/
ret = deregister_context(ce, ce->guc_id.id);
if (ret) {
spin_lock(&ce->guc_state.lock);
set_context_registered(ce);
clr_context_destroyed(ce);
spin_unlock(&ce->guc_state.lock);
/*
* As gt-pm is awake at function entry, intel_wakeref_put_async merely decrements
* the wakeref immediately but per function spec usage call this after unlock.
*/
intel_wakeref_put_async(>->wakeref);
}
return ret;
}
static void __guc_context_destroy(struct intel_context *ce)
{
GEM_BUG_ON(ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] ||
ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_HIGH] ||
ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] ||
ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_NORMAL]);
lrc_fini(ce);
intel_context_fini(ce);
if (intel_engine_is_virtual(ce->engine)) {
struct guc_virtual_engine *ve =
container_of(ce, typeof(*ve), context);
if (ve->base.breadcrumbs)
intel_breadcrumbs_put(ve->base.breadcrumbs);
kfree(ve);
} else {
intel_context_free(ce);
}
}
static void guc_flush_destroyed_contexts(struct intel_guc *guc)
{
struct intel_context *ce;
unsigned long flags;
GEM_BUG_ON(!submission_disabled(guc) &&
guc_submission_initialized(guc));
while (!list_empty(&guc->submission_state.destroyed_contexts)) {
spin_lock_irqsave(&guc->submission_state.lock, flags);
ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
struct intel_context,
destroyed_link);
if (ce)
list_del_init(&ce->destroyed_link);
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
if (!ce)
break;
release_guc_id(guc, ce);
__guc_context_destroy(ce);
}
}
static void deregister_destroyed_contexts(struct intel_guc *guc)
{
struct intel_context *ce;
unsigned long flags;
while (!list_empty(&guc->submission_state.destroyed_contexts)) {
spin_lock_irqsave(&guc->submission_state.lock, flags);
ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
struct intel_context,
destroyed_link);
if (ce)
list_del_init(&ce->destroyed_link);
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
if (!ce)
break;
if (guc_lrc_desc_unpin(ce)) {
/*
* This means GuC's CT link severed mid-way which could happen
* in suspend-resume corner cases. In this case, put the
* context back into the destroyed_contexts list which will
* get picked up on the next context deregistration event or
* purged in a GuC sanitization event (reset/unload/wedged/...).
*/
spin_lock_irqsave(&guc->submission_state.lock, flags);
list_add_tail(&ce->destroyed_link,
&guc->submission_state.destroyed_contexts);
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
/* Bail now since the list might never be emptied if h2gs fail */
break;
}
}
}
static void destroyed_worker_func(struct work_struct *w)
{
struct intel_guc *guc = container_of(w, struct intel_guc,
submission_state.destroyed_worker);
struct intel_gt *gt = guc_to_gt(guc);
intel_wakeref_t wakeref;
/*
* In rare cases we can get here via async context-free fence-signals that
* come very late in suspend flow or very early in resume flows. In these
* cases, GuC won't be ready but just skipping it here is fine as these
* pending-destroy-contexts get destroyed totally at GuC reset time at the
* end of suspend.. OR.. this worker can be picked up later on the next
* context destruction trigger after resume-completes
*/
if (!intel_guc_is_ready(guc))
return;
with_intel_gt_pm(gt, wakeref)
deregister_destroyed_contexts(guc);
}
static void guc_context_destroy(struct kref *kref)
{
struct intel_context *ce = container_of(kref, typeof(*ce), ref);
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
bool destroy;
/*
* If the guc_id is invalid this context has been stolen and we can free
* it immediately. Also can be freed immediately if the context is not
* registered with the GuC or the GuC is in the middle of a reset.
*/
spin_lock_irqsave(&guc->submission_state.lock, flags);
destroy = submission_disabled(guc) || context_guc_id_invalid(ce) ||
!ctx_id_mapped(guc, ce->guc_id.id);
if (likely(!destroy)) {
if (!list_empty(&ce->guc_id.link))
list_del_init(&ce->guc_id.link);
list_add_tail(&ce->destroyed_link,
&guc->submission_state.destroyed_contexts);
} else {
__release_guc_id(guc, ce);
}
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
if (unlikely(destroy)) {
__guc_context_destroy(ce);
return;
}
/*
* We use a worker to issue the H2G to deregister the context as we can
* take the GT PM for the first time which isn't allowed from an atomic
* context.
*/
queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker);
}
static int guc_context_alloc(struct intel_context *ce)
{
return lrc_alloc(ce, ce->engine);
}
static void __guc_context_set_prio(struct intel_guc *guc,
struct intel_context *ce)
{
if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
struct context_policy policy;
__guc_context_policy_start_klv(&policy, ce->guc_id.id);
__guc_context_policy_add_priority(&policy, ce->guc_state.prio);
__guc_context_set_context_policies(guc, &policy, true);
} else {
u32 action[] = {
INTEL_GUC_ACTION_V69_SET_CONTEXT_PRIORITY,
ce->guc_id.id,
ce->guc_state.prio,
};
guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
}
}
static void guc_context_set_prio(struct intel_guc *guc,
struct intel_context *ce,
u8 prio)
{
GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH ||
prio > GUC_CLIENT_PRIORITY_NORMAL);
lockdep_assert_held(&ce->guc_state.lock);
if (ce->guc_state.prio == prio || submission_disabled(guc) ||
!context_registered(ce)) {
ce->guc_state.prio = prio;
return;
}
ce->guc_state.prio = prio;
__guc_context_set_prio(guc, ce);
trace_intel_context_set_prio(ce);
}
static inline u8 map_i915_prio_to_guc_prio(int prio)
{
if (prio == I915_PRIORITY_NORMAL)
return GUC_CLIENT_PRIORITY_KMD_NORMAL;
else if (prio < I915_PRIORITY_NORMAL)
return GUC_CLIENT_PRIORITY_NORMAL;
else if (prio < I915_PRIORITY_DISPLAY)
return GUC_CLIENT_PRIORITY_HIGH;
else
return GUC_CLIENT_PRIORITY_KMD_HIGH;
}
static inline void add_context_inflight_prio(struct intel_context *ce,
u8 guc_prio)
{
lockdep_assert_held(&ce->guc_state.lock);
GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));
++ce->guc_state.prio_count[guc_prio];
/* Overflow protection */
GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);
}
static inline void sub_context_inflight_prio(struct intel_context *ce,
u8 guc_prio)
{
lockdep_assert_held(&ce->guc_state.lock);
GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));
/* Underflow protection */
GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);
--ce->guc_state.prio_count[guc_prio];
}
static inline void update_context_prio(struct intel_context *ce)
{
struct intel_guc *guc = &ce->engine->gt->uc.guc;
int i;
BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0);
BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL);
lockdep_assert_held(&ce->guc_state.lock);
for (i = 0; i < ARRAY_SIZE(ce->guc_state.prio_count); ++i) {
if (ce->guc_state.prio_count[i]) {
guc_context_set_prio(guc, ce, i);
break;
}
}
}
static inline bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio)
{
/* Lower value is higher priority */
return new_guc_prio < old_guc_prio;
}
static void add_to_context(struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
u8 new_guc_prio = map_i915_prio_to_guc_prio(rq_prio(rq));
GEM_BUG_ON(intel_context_is_child(ce));
GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI);
spin_lock(&ce->guc_state.lock);
list_move_tail(&rq->sched.link, &ce->guc_state.requests);
if (rq->guc_prio == GUC_PRIO_INIT) {
rq->guc_prio = new_guc_prio;
add_context_inflight_prio(ce, rq->guc_prio);
} else if (new_guc_prio_higher(rq->guc_prio, new_guc_prio)) {
sub_context_inflight_prio(ce, rq->guc_prio);
rq->guc_prio = new_guc_prio;
add_context_inflight_prio(ce, rq->guc_prio);
}
update_context_prio(ce);
spin_unlock(&ce->guc_state.lock);
}
static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce)
{
lockdep_assert_held(&ce->guc_state.lock);
if (rq->guc_prio != GUC_PRIO_INIT &&
rq->guc_prio != GUC_PRIO_FINI) {
sub_context_inflight_prio(ce, rq->guc_prio);
update_context_prio(ce);
}
rq->guc_prio = GUC_PRIO_FINI;
}
static void remove_from_context(struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
GEM_BUG_ON(intel_context_is_child(ce));
spin_lock_irq(&ce->guc_state.lock);
list_del_init(&rq->sched.link);
clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
/* Prevent further __await_execution() registering a cb, then flush */
set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
guc_prio_fini(rq, ce);
spin_unlock_irq(&ce->guc_state.lock);
atomic_dec(&ce->guc_id.ref);
i915_request_notify_execute_cb_imm(rq);
}
static const struct intel_context_ops guc_context_ops = {
.flags = COPS_RUNTIME_CYCLES,
.alloc = guc_context_alloc,
.close = guc_context_close,
.pre_pin = guc_context_pre_pin,
.pin = guc_context_pin,
.unpin = guc_context_unpin,
.post_unpin = guc_context_post_unpin,
.revoke = guc_context_revoke,
.cancel_request = guc_context_cancel_request,
.enter = intel_context_enter_engine,
.exit = intel_context_exit_engine,
.sched_disable = guc_context_sched_disable,
.update_stats = guc_context_update_stats,
.reset = lrc_reset,
.destroy = guc_context_destroy,
.create_virtual = guc_create_virtual,
.create_parallel = guc_create_parallel,
};
static void submit_work_cb(struct irq_work *wrk)
{
struct i915_request *rq = container_of(wrk, typeof(*rq), submit_work);
might_lock(&rq->engine->sched_engine->lock);
i915_sw_fence_complete(&rq->submit);
}
static void __guc_signal_context_fence(struct intel_context *ce)
{
struct i915_request *rq, *rn;
lockdep_assert_held(&ce->guc_state.lock);
if (!list_empty(&ce->guc_state.fences))
trace_intel_context_fence_release(ce);
/*
* Use an IRQ to ensure locking order of sched_engine->lock ->
* ce->guc_state.lock is preserved.
*/
list_for_each_entry_safe(rq, rn, &ce->guc_state.fences,
guc_fence_link) {
list_del(&rq->guc_fence_link);
irq_work_queue(&rq->submit_work);
}
INIT_LIST_HEAD(&ce->guc_state.fences);
}
static void guc_signal_context_fence(struct intel_context *ce)
{
unsigned long flags;
GEM_BUG_ON(intel_context_is_child(ce));
spin_lock_irqsave(&ce->guc_state.lock, flags);
clr_context_wait_for_deregister_to_register(ce);
__guc_signal_context_fence(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
}
static bool context_needs_register(struct intel_context *ce, bool new_guc_id)
{
return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) ||
!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id)) &&
!submission_disabled(ce_to_guc(ce));
}
static void guc_context_init(struct intel_context *ce)
{
const struct i915_gem_context *ctx;
int prio = I915_CONTEXT_DEFAULT_PRIORITY;
rcu_read_lock();
ctx = rcu_dereference(ce->gem_context);
if (ctx)
prio = ctx->sched.priority;
rcu_read_unlock();
ce->guc_state.prio = map_i915_prio_to_guc_prio(prio);
INIT_DELAYED_WORK(&ce->guc_state.sched_disable_delay_work,
__delay_sched_disable);
set_bit(CONTEXT_GUC_INIT, &ce->flags);
}
static int guc_request_alloc(struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
struct intel_guc *guc = ce_to_guc(ce);
unsigned long flags;
int ret;
GEM_BUG_ON(!intel_context_is_pinned(rq->context));
/*
* Flush enough space to reduce the likelihood of waiting after
* we start building the request - in which case we will just
* have to repeat work.
*/
rq->reserved_space += GUC_REQUEST_SIZE;
/*
* Note that after this point, we have committed to using
* this request as it is being used to both track the
* state of engine initialisation and liveness of the
* golden renderstate above. Think twice before you try
* to cancel/unwind this request now.
*/
/* Unconditionally invalidate GPU caches and TLBs. */
ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
if (ret)
return ret;
rq->reserved_space -= GUC_REQUEST_SIZE;
if (unlikely(!test_bit(CONTEXT_GUC_INIT, &ce->flags)))
guc_context_init(ce);
/*
* If the context gets closed while the execbuf is ongoing, the context
* close code will race with the below code to cancel the delayed work.
* If the context close wins the race and cancels the work, it will
* immediately call the sched disable (see guc_context_close), so there
* is a chance we can get past this check while the sched_disable code
* is being executed. To make sure that code completes before we check
* the status further down, we wait for the close process to complete.
* Else, this code path could send a request down thinking that the
* context is still in a schedule-enable mode while the GuC ends up
* dropping the request completely because the disable did go from the
* context_close path right to GuC just prior. In the event the CT is
* full, we could potentially need to wait up to 1.5 seconds.
*/
if (cancel_delayed_work_sync(&ce->guc_state.sched_disable_delay_work))
intel_context_sched_disable_unpin(ce);
else if (intel_context_is_closed(ce))
if (wait_for(context_close_done(ce), 1500))
guc_warn(guc, "timed out waiting on context sched close before realloc\n");
/*
* Call pin_guc_id here rather than in the pinning step as with
* dma_resv, contexts can be repeatedly pinned / unpinned trashing the
* guc_id and creating horrible race conditions. This is especially bad
* when guc_id are being stolen due to over subscription. By the time
* this function is reached, it is guaranteed that the guc_id will be
* persistent until the generated request is retired. Thus, sealing these
* race conditions. It is still safe to fail here if guc_id are
* exhausted and return -EAGAIN to the user indicating that they can try
* again in the future.
*
* There is no need for a lock here as the timeline mutex ensures at
* most one context can be executing this code path at once. The
* guc_id_ref is incremented once for every request in flight and
* decremented on each retire. When it is zero, a lock around the
* increment (in pin_guc_id) is needed to seal a race with unpin_guc_id.
*/
if (atomic_add_unless(&ce->guc_id.ref, 1, 0))
goto out;
ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */
if (unlikely(ret < 0))
return ret;
if (context_needs_register(ce, !!ret)) {
ret = try_context_registration(ce, true);
if (unlikely(ret)) { /* unwind */
if (ret == -EPIPE) {
disable_submission(guc);
goto out; /* GPU will be reset */
}
atomic_dec(&ce->guc_id.ref);
unpin_guc_id(guc, ce);
return ret;
}
}
clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags);
out:
/*
* We block all requests on this context if a G2H is pending for a
* schedule disable or context deregistration as the GuC will fail a
* schedule enable or context registration if either G2H is pending
* respectfully. Once a G2H returns, the fence is released that is
* blocking these requests (see guc_signal_context_fence).
*/
spin_lock_irqsave(&ce->guc_state.lock, flags);
if (context_wait_for_deregister_to_register(ce) ||
context_pending_disable(ce)) {
init_irq_work(&rq->submit_work, submit_work_cb);
i915_sw_fence_await(&rq->submit);
list_add_tail(&rq->guc_fence_link, &ce->guc_state.fences);
}
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
return 0;
}
static int guc_virtual_context_pre_pin(struct intel_context *ce,
struct i915_gem_ww_ctx *ww,
void **vaddr)
{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
return __guc_context_pre_pin(ce, engine, ww, vaddr);
}
static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr)
{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
int ret = __guc_context_pin(ce, engine, vaddr);
intel_engine_mask_t tmp, mask = ce->engine->mask;
if (likely(!ret))
for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
intel_engine_pm_get(engine);
return ret;
}
static void guc_virtual_context_unpin(struct intel_context *ce)
{
intel_engine_mask_t tmp, mask = ce->engine->mask;
struct intel_engine_cs *engine;
struct intel_guc *guc = ce_to_guc(ce);
GEM_BUG_ON(context_enabled(ce));
GEM_BUG_ON(intel_context_is_barrier(ce));
unpin_guc_id(guc, ce);
lrc_unpin(ce);
for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
intel_engine_pm_put_async(engine);
}
static void guc_virtual_context_enter(struct intel_context *ce)
{
intel_engine_mask_t tmp, mask = ce->engine->mask;
struct intel_engine_cs *engine;
for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
intel_engine_pm_get(engine);
intel_timeline_enter(ce->timeline);
}
static void guc_virtual_context_exit(struct intel_context *ce)
{
intel_engine_mask_t tmp, mask = ce->engine->mask;
struct intel_engine_cs *engine;
for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
intel_engine_pm_put(engine);
intel_timeline_exit(ce->timeline);
}
static int guc_virtual_context_alloc(struct intel_context *ce)
{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
return lrc_alloc(ce, engine);
}
static const struct intel_context_ops virtual_guc_context_ops = {
.flags = COPS_RUNTIME_CYCLES,
.alloc = guc_virtual_context_alloc,
.close = guc_context_close,
.pre_pin = guc_virtual_context_pre_pin,
.pin = guc_virtual_context_pin,
.unpin = guc_virtual_context_unpin,
.post_unpin = guc_context_post_unpin,
.revoke = guc_context_revoke,
.cancel_request = guc_context_cancel_request,
.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,
.sched_disable = guc_context_sched_disable,
.update_stats = guc_context_update_stats,
.destroy = guc_context_destroy,
.get_sibling = guc_virtual_get_sibling,
};
static int guc_parent_context_pin(struct intel_context *ce, void *vaddr)
{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
struct intel_guc *guc = ce_to_guc(ce);
int ret;
GEM_BUG_ON(!intel_context_is_parent(ce));
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
ret = pin_guc_id(guc, ce);
if (unlikely(ret < 0))
return ret;
return __guc_context_pin(ce, engine, vaddr);
}
static int guc_child_context_pin(struct intel_context *ce, void *vaddr)
{
struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
GEM_BUG_ON(!intel_context_is_child(ce));
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
__intel_context_pin(ce->parallel.parent);
return __guc_context_pin(ce, engine, vaddr);
}
static void guc_parent_context_unpin(struct intel_context *ce)
{
struct intel_guc *guc = ce_to_guc(ce);
GEM_BUG_ON(context_enabled(ce));
GEM_BUG_ON(intel_context_is_barrier(ce));
GEM_BUG_ON(!intel_context_is_parent(ce));
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
unpin_guc_id(guc, ce);
lrc_unpin(ce);
}
static void guc_child_context_unpin(struct intel_context *ce)
{
GEM_BUG_ON(context_enabled(ce));
GEM_BUG_ON(intel_context_is_barrier(ce));
GEM_BUG_ON(!intel_context_is_child(ce));
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
lrc_unpin(ce);
}
static void guc_child_context_post_unpin(struct intel_context *ce)
{
GEM_BUG_ON(!intel_context_is_child(ce));
GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent));
GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
lrc_post_unpin(ce);
intel_context_unpin(ce->parallel.parent);
}
static void guc_child_context_destroy(struct kref *kref)
{
struct intel_context *ce = container_of(kref, typeof(*ce), ref);
__guc_context_destroy(ce);
}
static const struct intel_context_ops virtual_parent_context_ops = {
.alloc = guc_virtual_context_alloc,
.close = guc_context_close,
.pre_pin = guc_context_pre_pin,
.pin = guc_parent_context_pin,
.unpin = guc_parent_context_unpin,
.post_unpin = guc_context_post_unpin,
.revoke = guc_context_revoke,
.cancel_request = guc_context_cancel_request,
.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,
.sched_disable = guc_context_sched_disable,
.destroy = guc_context_destroy,
.get_sibling = guc_virtual_get_sibling,
};
static const struct intel_context_ops virtual_child_context_ops = {
.alloc = guc_virtual_context_alloc,
.pre_pin = guc_context_pre_pin,
.pin = guc_child_context_pin,
.unpin = guc_child_context_unpin,
.post_unpin = guc_child_context_post_unpin,
.cancel_request = guc_context_cancel_request,
.enter = guc_virtual_context_enter,
.exit = guc_virtual_context_exit,
.destroy = guc_child_context_destroy,
.get_sibling = guc_virtual_get_sibling,
};
/*
* The below override of the breadcrumbs is enabled when the user configures a
* context for parallel submission (multi-lrc, parent-child).
*
* The overridden breadcrumbs implements an algorithm which allows the GuC to
* safely preempt all the hw contexts configured for parallel submission
* between each BB. The contract between the i915 and GuC is if the parent
* context can be preempted, all the children can be preempted, and the GuC will
* always try to preempt the parent before the children. A handshake between the
* parent / children breadcrumbs ensures the i915 holds up its end of the deal
* creating a window to preempt between each set of BBs.
*/
static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
u64 offset, u32 len,
const unsigned int flags);
static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
u64 offset, u32 len,
const unsigned int flags);
static u32 *
emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
u32 *cs);
static u32 *
emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
u32 *cs);
static struct intel_context *
guc_create_parallel(struct intel_engine_cs **engines,
unsigned int num_siblings,
unsigned int width)
{
struct intel_engine_cs **siblings = NULL;
struct intel_context *parent = NULL, *ce, *err;
int i, j;
siblings = kmalloc_array(num_siblings,
sizeof(*siblings),
GFP_KERNEL);
if (!siblings)
return ERR_PTR(-ENOMEM);
for (i = 0; i < width; ++i) {
for (j = 0; j < num_siblings; ++j)
siblings[j] = engines[i * num_siblings + j];
ce = intel_engine_create_virtual(siblings, num_siblings,
FORCE_VIRTUAL);
if (IS_ERR(ce)) {
err = ERR_CAST(ce);
goto unwind;
}
if (i == 0) {
parent = ce;
parent->ops = &virtual_parent_context_ops;
} else {
ce->ops = &virtual_child_context_ops;
intel_context_bind_parent_child(parent, ce);
}
}
parent->parallel.fence_context = dma_fence_context_alloc(1);
parent->engine->emit_bb_start =
emit_bb_start_parent_no_preempt_mid_batch;
parent->engine->emit_fini_breadcrumb =
emit_fini_breadcrumb_parent_no_preempt_mid_batch;
parent->engine->emit_fini_breadcrumb_dw =
12 + 4 * parent->parallel.number_children;
for_each_child(parent, ce) {
ce->engine->emit_bb_start =
emit_bb_start_child_no_preempt_mid_batch;
ce->engine->emit_fini_breadcrumb =
emit_fini_breadcrumb_child_no_preempt_mid_batch;
ce->engine->emit_fini_breadcrumb_dw = 16;
}
kfree(siblings);
return parent;
unwind:
if (parent)
intel_context_put(parent);
kfree(siblings);
return err;
}
static bool
guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b)
{
struct intel_engine_cs *sibling;
intel_engine_mask_t tmp, mask = b->engine_mask;
bool result = false;
for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)
result |= intel_engine_irq_enable(sibling);
return result;
}
static void
guc_irq_disable_breadcrumbs(struct intel_breadcrumbs *b)
{
struct intel_engine_cs *sibling;
intel_engine_mask_t tmp, mask = b->engine_mask;
for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)
intel_engine_irq_disable(sibling);
}
static void guc_init_breadcrumbs(struct intel_engine_cs *engine)
{
int i;
/*
* In GuC submission mode we do not know which physical engine a request
* will be scheduled on, this creates a problem because the breadcrumb
* interrupt is per physical engine. To work around this we attach
* requests and direct all breadcrumb interrupts to the first instance
* of an engine per class. In addition all breadcrumb interrupts are
* enabled / disabled across an engine class in unison.
*/
for (i = 0; i < MAX_ENGINE_INSTANCE; ++i) {
struct intel_engine_cs *sibling =
engine->gt->engine_class[engine->class][i];
if (sibling) {
if (engine->breadcrumbs != sibling->breadcrumbs) {
intel_breadcrumbs_put(engine->breadcrumbs);
engine->breadcrumbs =
intel_breadcrumbs_get(sibling->breadcrumbs);
}
break;
}
}
if (engine->breadcrumbs) {
engine->breadcrumbs->engine_mask |= engine->mask;
engine->breadcrumbs->irq_enable = guc_irq_enable_breadcrumbs;
engine->breadcrumbs->irq_disable = guc_irq_disable_breadcrumbs;
}
}
static void guc_bump_inflight_request_prio(struct i915_request *rq,
int prio)
{
struct intel_context *ce = request_to_scheduling_context(rq);
u8 new_guc_prio = map_i915_prio_to_guc_prio(prio);
/* Short circuit function */
if (prio < I915_PRIORITY_NORMAL)
return;
spin_lock(&ce->guc_state.lock);
if (rq->guc_prio == GUC_PRIO_FINI)
goto exit;
if (!new_guc_prio_higher(rq->guc_prio, new_guc_prio))
goto exit;
if (rq->guc_prio != GUC_PRIO_INIT)
sub_context_inflight_prio(ce, rq->guc_prio);
rq->guc_prio = new_guc_prio;
add_context_inflight_prio(ce, rq->guc_prio);
update_context_prio(ce);
exit:
spin_unlock(&ce->guc_state.lock);
}
static void guc_retire_inflight_request_prio(struct i915_request *rq)
{
struct intel_context *ce = request_to_scheduling_context(rq);
spin_lock(&ce->guc_state.lock);
guc_prio_fini(rq, ce);
spin_unlock(&ce->guc_state.lock);
}
static void sanitize_hwsp(struct intel_engine_cs *engine)
{
struct intel_timeline *tl;
list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
intel_timeline_reset_seqno(tl);
}
static void guc_sanitize(struct intel_engine_cs *engine)
{
/*
* Poison residual state on resume, in case the suspend didn't!
*
* We have to assume that across suspend/resume (or other loss
* of control) that the contents of our pinned buffers has been
* lost, replaced by garbage. Since this doesn't always happen,
* let's poison such state so that we more quickly spot when
* we falsely assume it has been preserved.
*/
if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
/*
* The kernel_context HWSP is stored in the status_page. As above,
* that may be lost on resume/initialisation, and so we need to
* reset the value in the HWSP.
*/
sanitize_hwsp(engine);
/* And scrub the dirty cachelines for the HWSP */
drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE);
intel_engine_reset_pinned_contexts(engine);
}
static void setup_hwsp(struct intel_engine_cs *engine)
{
intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
ENGINE_WRITE_FW(engine,
RING_HWS_PGA,
i915_ggtt_offset(engine->status_page.vma));
}
static void start_engine(struct intel_engine_cs *engine)
{
ENGINE_WRITE_FW(engine,
RING_MODE_GEN7,
_MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
ENGINE_POSTING_READ(engine, RING_MI_MODE);
}
static int guc_resume(struct intel_engine_cs *engine)
{
assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
intel_mocs_init_engine(engine);
intel_breadcrumbs_reset(engine->breadcrumbs);
setup_hwsp(engine);
start_engine(engine);
if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
xehp_enable_ccs_engines(engine);
return 0;
}
static bool guc_sched_engine_disabled(struct i915_sched_engine *sched_engine)
{
return !sched_engine->tasklet.callback;
}
static void guc_set_default_submission(struct intel_engine_cs *engine)
{
engine->submit_request = guc_submit_request;
}
static inline int guc_kernel_context_pin(struct intel_guc *guc,
struct intel_context *ce)
{
int ret;
/*
* Note: we purposefully do not check the returns below because
* the registration can only fail if a reset is just starting.
* This is called at the end of reset so presumably another reset
* isn't happening and even it did this code would be run again.
*/
if (context_guc_id_invalid(ce)) {
ret = pin_guc_id(guc, ce);
if (ret < 0)
return ret;
}
if (!test_bit(CONTEXT_GUC_INIT, &ce->flags))
guc_context_init(ce);
ret = try_context_registration(ce, true);
if (ret)
unpin_guc_id(guc, ce);
return ret;
}
static inline int guc_init_submission(struct intel_guc *guc)
{
struct intel_gt *gt = guc_to_gt(guc);
struct intel_engine_cs *engine;
enum intel_engine_id id;
/* make sure all descriptors are clean... */
xa_destroy(&guc->context_lookup);
/*
* A reset might have occurred while we had a pending stalled request,
* so make sure we clean that up.
*/
guc->stalled_request = NULL;
guc->submission_stall_reason = STALL_NONE;
/*
* Some contexts might have been pinned before we enabled GuC
* submission, so we need to add them to the GuC bookeeping.
* Also, after a reset the of the GuC we want to make sure that the
* information shared with GuC is properly reset. The kernel LRCs are
* not attached to the gem_context, so they need to be added separately.
*/
for_each_engine(engine, gt, id) {
struct intel_context *ce;
list_for_each_entry(ce, &engine->pinned_contexts_list,
pinned_contexts_link) {
int ret = guc_kernel_context_pin(guc, ce);
if (ret) {
/* No point in trying to clean up as i915 will wedge on failure */
return ret;
}
}
}
return 0;
}
static void guc_release(struct intel_engine_cs *engine)
{
engine->sanitize = NULL; /* no longer in control, nothing to sanitize */
intel_engine_cleanup_common(engine);
lrc_fini_wa_ctx(engine);
}
static void virtual_guc_bump_serial(struct intel_engine_cs *engine)
{
struct intel_engine_cs *e;
intel_engine_mask_t tmp, mask = engine->mask;
for_each_engine_masked(e, engine->gt, mask, tmp)
e->serial++;
}
static void guc_default_vfuncs(struct intel_engine_cs *engine)
{
/* Default vfuncs which can be overridden by each engine. */
engine->resume = guc_resume;
engine->cops = &guc_context_ops;
engine->request_alloc = guc_request_alloc;
engine->add_active_request = add_to_context;
engine->remove_active_request = remove_from_context;
engine->sched_engine->schedule = i915_schedule;
engine->reset.prepare = guc_engine_reset_prepare;
engine->reset.rewind = guc_rewind_nop;
engine->reset.cancel = guc_reset_nop;
engine->reset.finish = guc_reset_nop;
engine->emit_flush = gen8_emit_flush_xcs;
engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
if (GRAPHICS_VER(engine->i915) >= 12) {
engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
engine->emit_flush = gen12_emit_flush_xcs;
}
engine->set_default_submission = guc_set_default_submission;
engine->busyness = guc_engine_busyness;
engine->flags |= I915_ENGINE_SUPPORTS_STATS;
engine->flags |= I915_ENGINE_HAS_PREEMPTION;
engine->flags |= I915_ENGINE_HAS_TIMESLICES;
/* Wa_14014475959:dg2 */
if (engine->class == COMPUTE_CLASS)
if (IS_GFX_GT_IP_STEP(engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
IS_DG2(engine->i915))
engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
/* Wa_16019325821 */
/* Wa_14019159160 */
if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) &&
IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))
engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;
/*
* TODO: GuC supports timeslicing and semaphores as well, but they're
* handled by the firmware so some minor tweaks are required before
* enabling.
*
* engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
*/
engine->emit_bb_start = gen8_emit_bb_start;
if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
engine->emit_bb_start = xehp_emit_bb_start;
}
static void rcs_submission_override(struct intel_engine_cs *engine)
{
switch (GRAPHICS_VER(engine->i915)) {
case 12:
engine->emit_flush = gen12_emit_flush_rcs;
engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
break;
case 11:
engine->emit_flush = gen11_emit_flush_rcs;
engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
break;
default:
engine->emit_flush = gen8_emit_flush_rcs;
engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
break;
}
}
static inline void guc_default_irqs(struct intel_engine_cs *engine)
{
engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT;
intel_engine_set_irq_handler(engine, cs_irq_handler);
}
static void guc_sched_engine_destroy(struct kref *kref)
{
struct i915_sched_engine *sched_engine =
container_of(kref, typeof(*sched_engine), ref);
struct intel_guc *guc = sched_engine->private_data;
guc->sched_engine = NULL;
tasklet_kill(&sched_engine->tasklet); /* flush the callback */
kfree(sched_engine);
}
int intel_guc_submission_setup(struct intel_engine_cs *engine)
{
struct drm_i915_private *i915 = engine->i915;
struct intel_guc *guc = gt_to_guc(engine->gt);
/*
* The setup relies on several assumptions (e.g. irqs always enabled)
* that are only valid on gen11+
*/
GEM_BUG_ON(GRAPHICS_VER(i915) < 11);
if (!guc->sched_engine) {
guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
if (!guc->sched_engine)
return -ENOMEM;
guc->sched_engine->schedule = i915_schedule;
guc->sched_engine->disabled = guc_sched_engine_disabled;
guc->sched_engine->private_data = guc;
guc->sched_engine->destroy = guc_sched_engine_destroy;
guc->sched_engine->bump_inflight_request_prio =
guc_bump_inflight_request_prio;
guc->sched_engine->retire_inflight_request_prio =
guc_retire_inflight_request_prio;
tasklet_setup(&guc->sched_engine->tasklet,
guc_submission_tasklet);
}
i915_sched_engine_put(engine->sched_engine);
engine->sched_engine = i915_sched_engine_get(guc->sched_engine);
guc_default_vfuncs(engine);
guc_default_irqs(engine);
guc_init_breadcrumbs(engine);
if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)
rcs_submission_override(engine);
lrc_init_wa_ctx(engine);
/* Finally, take ownership and responsibility for cleanup! */
engine->sanitize = guc_sanitize;
engine->release = guc_release;
return 0;
}
struct scheduling_policy {
/* internal data */
u32 max_words, num_words;
u32 count;
/* API data */
struct guc_update_scheduling_policy h2g;
};
static u32 __guc_scheduling_policy_action_size(struct scheduling_policy *policy)
{
u32 *start = (void *)&policy->h2g;
u32 *end = policy->h2g.data + policy->num_words;
size_t delta = end - start;
return delta;
}
static struct scheduling_policy *__guc_scheduling_policy_start_klv(struct scheduling_policy *policy)
{
policy->h2g.header.action = INTEL_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV;
policy->max_words = ARRAY_SIZE(policy->h2g.data);
policy->num_words = 0;
policy->count = 0;
return policy;
}
static void __guc_scheduling_policy_add_klv(struct scheduling_policy *policy,
u32 action, u32 *data, u32 len)
{
u32 *klv_ptr = policy->h2g.data + policy->num_words;
GEM_BUG_ON((policy->num_words + 1 + len) > policy->max_words);
*(klv_ptr++) = FIELD_PREP(GUC_KLV_0_KEY, action) |
FIELD_PREP(GUC_KLV_0_LEN, len);
memcpy(klv_ptr, data, sizeof(u32) * len);
policy->num_words += 1 + len;
policy->count++;
}
static int __guc_action_set_scheduling_policies(struct intel_guc *guc,
struct scheduling_policy *policy)
{
int ret;
ret = intel_guc_send(guc, (u32 *)&policy->h2g,
__guc_scheduling_policy_action_size(policy));
if (ret < 0) {
guc_probe_error(guc, "Failed to configure global scheduling policies: %pe!\n",
ERR_PTR(ret));
return ret;
}
if (ret != policy->count) {
guc_warn(guc, "global scheduler policy processed %d of %d KLVs!",
ret, policy->count);
if (ret > policy->count)
return -EPROTO;
}
return 0;
}
static int guc_init_global_schedule_policy(struct intel_guc *guc)
{
struct scheduling_policy policy;
struct intel_gt *gt = guc_to_gt(guc);
intel_wakeref_t wakeref;
int ret;
if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))
return 0;
__guc_scheduling_policy_start_klv(&policy);
with_intel_runtime_pm(>->i915->runtime_pm, wakeref) {
u32 yield[] = {
GLOBAL_SCHEDULE_POLICY_RC_YIELD_DURATION,
GLOBAL_SCHEDULE_POLICY_RC_YIELD_RATIO,
};
__guc_scheduling_policy_add_klv(&policy,
GUC_SCHEDULING_POLICIES_KLV_ID_RENDER_COMPUTE_YIELD,
yield, ARRAY_SIZE(yield));
ret = __guc_action_set_scheduling_policies(guc, &policy);
}
return ret;
}
static void guc_route_semaphores(struct intel_guc *guc, bool to_guc)
{
struct intel_gt *gt = guc_to_gt(guc);
u32 val;
if (GRAPHICS_VER(gt->i915) < 12)
return;
if (to_guc)
val = GUC_SEM_INTR_ROUTE_TO_GUC | GUC_SEM_INTR_ENABLE_ALL;
else
val = 0;
intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, val);
}
int intel_guc_submission_enable(struct intel_guc *guc)
{
int ret;
/* Semaphore interrupt enable and route to GuC */
guc_route_semaphores(guc, true);
ret = guc_init_submission(guc);
if (ret)
goto fail_sem;
ret = guc_init_engine_stats(guc);
if (ret)
goto fail_sem;
ret = guc_init_global_schedule_policy(guc);
if (ret)
goto fail_stats;
return 0;
fail_stats:
guc_fini_engine_stats(guc);
fail_sem:
guc_route_semaphores(guc, false);
return ret;
}
/* Note: By the time we're here, GuC may have already been reset */
void intel_guc_submission_disable(struct intel_guc *guc)
{
guc_cancel_busyness_worker(guc);
/* Semaphore interrupt disable and route to host */
guc_route_semaphores(guc, false);
}
static bool __guc_submission_supported(struct intel_guc *guc)
{
/* GuC submission is unavailable for pre-Gen11 */
return intel_guc_is_supported(guc) &&
GRAPHICS_VER(guc_to_i915(guc)) >= 11;
}
static bool __guc_submission_selected(struct intel_guc *guc)
{
struct drm_i915_private *i915 = guc_to_i915(guc);
if (!intel_guc_submission_is_supported(guc))
return false;
return i915->params.enable_guc & ENABLE_GUC_SUBMISSION;
}
int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc)
{
return guc->submission_state.num_guc_ids - NUMBER_MULTI_LRC_GUC_ID(guc);
}
/*
* This default value of 33 milisecs (+1 milisec round up) ensures 30fps or higher
* workloads are able to enjoy the latency reduction when delaying the schedule-disable
* operation. This matches the 30fps game-render + encode (real world) workload this
* knob was tested against.
*/
#define SCHED_DISABLE_DELAY_MS 34
/*
* A threshold of 75% is a reasonable starting point considering that real world apps
* generally don't get anywhere near this.
*/
#define NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(__guc) \
(((intel_guc_sched_disable_gucid_threshold_max(guc)) * 3) / 4)
void intel_guc_submission_init_early(struct intel_guc *guc)
{
xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);
spin_lock_init(&guc->submission_state.lock);
INIT_LIST_HEAD(&guc->submission_state.guc_id_list);
ida_init(&guc->submission_state.guc_ids);
INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts);
INIT_WORK(&guc->submission_state.destroyed_worker,
destroyed_worker_func);
INIT_WORK(&guc->submission_state.reset_fail_worker,
reset_fail_worker_func);
spin_lock_init(&guc->timestamp.lock);
INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping);
guc->submission_state.sched_disable_delay_ms = SCHED_DISABLE_DELAY_MS;
guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID;
guc->submission_state.sched_disable_gucid_threshold =
NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(guc);
guc->submission_supported = __guc_submission_supported(guc);
guc->submission_selected = __guc_submission_selected(guc);
}
static inline struct intel_context *
g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)
{
struct intel_context *ce;
if (unlikely(ctx_id >= GUC_MAX_CONTEXT_ID)) {
guc_err(guc, "Invalid ctx_id %u\n", ctx_id);
return NULL;
}
ce = __get_context(guc, ctx_id);
if (unlikely(!ce)) {
guc_err(guc, "Context is NULL, ctx_id %u\n", ctx_id);
return NULL;
}
if (unlikely(intel_context_is_child(ce))) {
guc_err(guc, "Context is child, ctx_id %u\n", ctx_id);
return NULL;
}
return ce;
}
static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno)
{
struct intel_guc_tlb_wait *wait;
unsigned long flags;
xa_lock_irqsave(&guc->tlb_lookup, flags);
wait = xa_load(&guc->tlb_lookup, seqno);
if (wait)
wake_up(&wait->wq);
else
guc_dbg(guc,
"Stale TLB invalidation response with seqno %d\n", seqno);
xa_unlock_irqrestore(&guc->tlb_lookup, flags);
}
int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
const u32 *payload, u32 len)
{
if (len < 1)
return -EPROTO;
wait_wake_outstanding_tlb_g2h(guc, payload[0]);
return 0;
}
static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout)
{
/*
* This is equivalent to wait_woken() with the exception that
* we do not wake up early if the kthread task has been completed.
* As we are called from page reclaim in any task context,
* we may be invoked from stopped kthreads, but we *must*
* complete the wait from the HW.
*/
do {
set_current_state(TASK_UNINTERRUPTIBLE);
if (wq_entry->flags & WQ_FLAG_WOKEN)
break;
timeout = schedule_timeout(timeout);
} while (timeout);
/* See wait_woken() and woken_wake_function() */
__set_current_state(TASK_RUNNING);
smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN);
return timeout;
}
static bool intel_gt_is_enabled(const struct intel_gt *gt)
{
/* Check if GT is wedged or suspended */
if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915))
return false;
return true;
}
static int guc_send_invalidate_tlb(struct intel_guc *guc,
enum intel_guc_tlb_invalidation_type type)
{
struct intel_guc_tlb_wait _wq, *wq = &_wq;
struct intel_gt *gt = guc_to_gt(guc);
DEFINE_WAIT_FUNC(wait, woken_wake_function);
int err;
u32 seqno;
u32 action[] = {
INTEL_GUC_ACTION_TLB_INVALIDATION,
0,
REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_TYPE_MASK, type) |
REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_MODE_MASK,
INTEL_GUC_TLB_INVAL_MODE_HEAVY) |
INTEL_GUC_TLB_INVAL_FLUSH_CACHE,
};
u32 size = ARRAY_SIZE(action);
/*
* Early guard against GT enablement. TLB invalidation should not be
* attempted if the GT is disabled due to suspend/wedge.
*/
if (!intel_gt_is_enabled(gt))
return -EINVAL;
init_waitqueue_head(&_wq.wq);
if (xa_alloc_cyclic_irq(&guc->tlb_lookup, &seqno, wq,
xa_limit_32b, &guc->next_seqno,
GFP_ATOMIC | __GFP_NOWARN) < 0) {
/* Under severe memory pressure? Serialise TLB allocations */
xa_lock_irq(&guc->tlb_lookup);
wq = xa_load(&guc->tlb_lookup, guc->serial_slot);
wait_event_lock_irq(wq->wq,
!READ_ONCE(wq->busy),
guc->tlb_lookup.xa_lock);
/*
* Update wq->busy under lock to ensure only one waiter can
* issue the TLB invalidation command using the serial slot at a
* time. The condition is set to true before releasing the lock
* so that other caller continue to wait until woken up again.
*/
wq->busy = true;
xa_unlock_irq(&guc->tlb_lookup);
seqno = guc->serial_slot;
}
action[1] = seqno;
add_wait_queue(&wq->wq, &wait);
/* This is a critical reclaim path and thus we must loop here. */
err = intel_guc_send_busy_loop(guc, action, size, G2H_LEN_DW_INVALIDATE_TLB, true);
if (err)
goto out;
/*
* Late guard against GT enablement. It is not an error for the TLB
* invalidation to time out if the GT is disabled during the process
* due to suspend/wedge. In fact, the TLB invalidation is cancelled
* in this case.
*/
if (!must_wait_woken(&wait, intel_guc_ct_max_queue_time_jiffies()) &&
intel_gt_is_enabled(gt)) {
guc_err(guc,
"TLB invalidation response timed out for seqno %u\n", seqno);
err = -ETIME;
}
out:
remove_wait_queue(&wq->wq, &wait);
if (seqno != guc->serial_slot)
xa_erase_irq(&guc->tlb_lookup, seqno);
return err;
}
/* Send a H2G command to invalidate the TLBs at engine level and beyond. */
int intel_guc_invalidate_tlb_engines(struct intel_guc *guc)
{
return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_ENGINES);
}
/* Send a H2G command to invalidate the GuC's internal TLB. */
int intel_guc_invalidate_tlb_guc(struct intel_guc *guc)
{
return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_GUC);
}
int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
const u32 *msg,
u32 len)
{
struct intel_context *ce;
u32 ctx_id;
if (unlikely(len < 1)) {
guc_err(guc, "Invalid length %u\n", len);
return -EPROTO;
}
ctx_id = msg[0];
ce = g2h_context_lookup(guc, ctx_id);
if (unlikely(!ce))
return -EPROTO;
trace_intel_context_deregister_done(ce);
#ifdef CONFIG_DRM_I915_SELFTEST
if (unlikely(ce->drop_deregister)) {
ce->drop_deregister = false;
return 0;
}
#endif
if (context_wait_for_deregister_to_register(ce)) {
struct intel_runtime_pm *runtime_pm =
&ce->engine->gt->i915->runtime_pm;
intel_wakeref_t wakeref;
/*
* Previous owner of this guc_id has been deregistered, now safe
* register this context.
*/
with_intel_runtime_pm(runtime_pm, wakeref)
register_context(ce, true);
guc_signal_context_fence(ce);
intel_context_put(ce);
} else if (context_destroyed(ce)) {
/* Context has been destroyed */
intel_gt_pm_put_async_untracked(guc_to_gt(guc));
release_guc_id(guc, ce);
__guc_context_destroy(ce);
}
decr_outstanding_submission_g2h(guc);
return 0;
}
int intel_guc_sched_done_process_msg(struct intel_guc *guc,
const u32 *msg,
u32 len)
{
struct intel_context *ce;
unsigned long flags;
u32 ctx_id;
if (unlikely(len < 2)) {
guc_err(guc, "Invalid length %u\n", len);
return -EPROTO;
}
ctx_id = msg[0];
ce = g2h_context_lookup(guc, ctx_id);
if (unlikely(!ce))
return -EPROTO;
if (unlikely(context_destroyed(ce) ||
(!context_pending_enable(ce) &&
!context_pending_disable(ce)))) {
guc_err(guc, "Bad context sched_state 0x%x, ctx_id %u\n",
ce->guc_state.sched_state, ctx_id);
return -EPROTO;
}
trace_intel_context_sched_done(ce);
if (context_pending_enable(ce)) {
#ifdef CONFIG_DRM_I915_SELFTEST
if (unlikely(ce->drop_schedule_enable)) {
ce->drop_schedule_enable = false;
return 0;
}
#endif
spin_lock_irqsave(&ce->guc_state.lock, flags);
clr_context_pending_enable(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
} else if (context_pending_disable(ce)) {
bool banned;
#ifdef CONFIG_DRM_I915_SELFTEST
if (unlikely(ce->drop_schedule_disable)) {
ce->drop_schedule_disable = false;
return 0;
}
#endif
/*
* Unpin must be done before __guc_signal_context_fence,
* otherwise a race exists between the requests getting
* submitted + retired before this unpin completes resulting in
* the pin_count going to zero and the context still being
* enabled.
*/
intel_context_sched_disable_unpin(ce);
spin_lock_irqsave(&ce->guc_state.lock, flags);
banned = context_banned(ce);
clr_context_banned(ce);
clr_context_pending_disable(ce);
__guc_signal_context_fence(ce);
guc_blocked_fence_complete(ce);
spin_unlock_irqrestore(&ce->guc_state.lock, flags);
if (banned) {
guc_cancel_context_requests(ce);
intel_engine_signal_breadcrumbs(ce->engine);
}
}
decr_outstanding_submission_g2h(guc);
intel_context_put(ce);
return 0;
}
static void capture_error_state(struct intel_guc *guc,
struct intel_context *ce)
{
struct intel_gt *gt = guc_to_gt(guc);
struct drm_i915_private *i915 = gt->i915;
intel_wakeref_t wakeref;
intel_engine_mask_t engine_mask;
if (intel_engine_is_virtual(ce->engine)) {
struct intel_engine_cs *e;
intel_engine_mask_t tmp, virtual_mask = ce->engine->mask;
engine_mask = 0;
for_each_engine_masked(e, ce->engine->gt, virtual_mask, tmp) {
bool match = intel_guc_capture_is_matching_engine(gt, ce, e);
if (match) {
intel_engine_set_hung_context(e, ce);
engine_mask |= e->mask;
i915_increase_reset_engine_count(&i915->gpu_error,
e);
}
}
if (!engine_mask) {
guc_warn(guc, "No matching physical engine capture for virtual engine context 0x%04X / %s",
ce->guc_id.id, ce->engine->name);
engine_mask = ~0U;
}
} else {
intel_engine_set_hung_context(ce->engine, ce);
engine_mask = ce->engine->mask;
i915_increase_reset_engine_count(&i915->gpu_error, ce->engine);
}
with_intel_runtime_pm(&i915->runtime_pm, wakeref)
i915_capture_error_state(gt, engine_mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE);
}
static void guc_context_replay(struct intel_context *ce)
{
struct i915_sched_engine *sched_engine = ce->engine->sched_engine;
__guc_reset_context(ce, ce->engine->mask);
tasklet_hi_schedule(&sched_engine->tasklet);
}
static void guc_handle_context_reset(struct intel_guc *guc,
struct intel_context *ce)
{
bool capture = intel_context_is_schedulable(ce);
trace_intel_context_reset(ce);
guc_dbg(guc, "%s context reset notification: 0x%04X on %s, exiting = %s, banned = %s\n",
capture ? "Got" : "Ignoring",
ce->guc_id.id, ce->engine->name,
str_yes_no(intel_context_is_exiting(ce)),
str_yes_no(intel_context_is_banned(ce)));
if (capture) {
capture_error_state(guc, ce);
guc_context_replay(ce);
}
}
int intel_guc_context_reset_process_msg(struct intel_guc *guc,
const u32 *msg, u32 len)
{
struct intel_context *ce;
unsigned long flags;
int ctx_id;
if (unlikely(len != 1)) {
guc_err(guc, "Invalid length %u", len);
return -EPROTO;
}
ctx_id = msg[0];
/*
* The context lookup uses the xarray but lookups only require an RCU lock
* not the full spinlock. So take the lock explicitly and keep it until the
* context has been reference count locked to ensure it can't be destroyed
* asynchronously until the reset is done.
*/
xa_lock_irqsave(&guc->context_lookup, flags);
ce = g2h_context_lookup(guc, ctx_id);
if (ce)
intel_context_get(ce);
xa_unlock_irqrestore(&guc->context_lookup, flags);
if (unlikely(!ce))
return -EPROTO;
guc_handle_context_reset(guc, ce);
intel_context_put(ce);
return 0;
}
int intel_guc_error_capture_process_msg(struct intel_guc *guc,
const u32 *msg, u32 len)
{
u32 status;
if (unlikely(len != 1)) {
guc_dbg(guc, "Invalid length %u", len);
return -EPROTO;
}
status = msg[0] & INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK;
if (status == INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)
guc_warn(guc, "No space for error capture");
intel_guc_capture_process(guc);
return 0;
}
struct intel_engine_cs *
intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance)
{
struct intel_gt *gt = guc_to_gt(guc);
u8 engine_class = guc_class_to_engine_class(guc_class);
/* Class index is checked in class converter */
GEM_BUG_ON(instance > MAX_ENGINE_INSTANCE);
return gt->engine_class[engine_class][instance];
}
static void reset_fail_worker_func(struct work_struct *w)
{
struct intel_guc *guc = container_of(w, struct intel_guc,
submission_state.reset_fail_worker);
struct intel_gt *gt = guc_to_gt(guc);
intel_engine_mask_t reset_fail_mask;
unsigned long flags;
spin_lock_irqsave(&guc->submission_state.lock, flags);
reset_fail_mask = guc->submission_state.reset_fail_mask;
guc->submission_state.reset_fail_mask = 0;
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
if (likely(reset_fail_mask)) {
struct intel_engine_cs *engine;
enum intel_engine_id id;
/*
* GuC is toast at this point - it dead loops after sending the failed
* reset notification. So need to manually determine the guilty context.
* Note that it should be reliable to do this here because the GuC is
* toast and will not be scheduling behind the KMD's back.
*/
for_each_engine_masked(engine, gt, reset_fail_mask, id)
intel_guc_find_hung_context(engine);
intel_gt_handle_error(gt, reset_fail_mask,
I915_ERROR_CAPTURE,
"GuC failed to reset engine mask=0x%x",
reset_fail_mask);
}
}
int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
const u32 *msg, u32 len)
{
struct intel_engine_cs *engine;
u8 guc_class, instance;
u32 reason;
unsigned long flags;
if (unlikely(len != 3)) {
guc_err(guc, "Invalid length %u", len);
return -EPROTO;
}
guc_class = msg[0];
instance = msg[1];
reason = msg[2];
engine = intel_guc_lookup_engine(guc, guc_class, instance);
if (unlikely(!engine)) {
guc_err(guc, "Invalid engine %d:%d", guc_class, instance);
return -EPROTO;
}
/*
* This is an unexpected failure of a hardware feature. So, log a real
* error message not just the informational that comes with the reset.
*/
guc_err(guc, "Engine reset failed on %d:%d (%s) because 0x%08X",
guc_class, instance, engine->name, reason);
spin_lock_irqsave(&guc->submission_state.lock, flags);
guc->submission_state.reset_fail_mask |= engine->mask;
spin_unlock_irqrestore(&guc->submission_state.lock, flags);
/*
* A GT reset flushes this worker queue (G2H handler) so we must use
* another worker to trigger a GT reset.
*/
queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker);
return 0;
}
void intel_guc_find_hung_context(struct intel_engine_cs *engine)
{
struct intel_guc *guc = gt_to_guc(engine->gt);
struct intel_context *ce;
struct i915_request *rq;
unsigned long index;
unsigned long flags;
/* Reset called during driver load? GuC not yet initialised! */
if (unlikely(!guc_submission_initialized(guc)))
return;
xa_lock_irqsave(&guc->context_lookup, flags);
xa_for_each(&guc->context_lookup, index, ce) {
bool found;
if (!kref_get_unless_zero(&ce->ref))
continue;
xa_unlock(&guc->context_lookup);
if (!intel_context_is_pinned(ce))
goto next;
if (intel_engine_is_virtual(ce->engine)) {
if (!(ce->engine->mask & engine->mask))
goto next;
} else {
if (ce->engine != engine)
goto next;
}
found = false;
spin_lock(&ce->guc_state.lock);
list_for_each_entry(rq, &ce->guc_state.requests, sched.link) {
if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE)
continue;
found = true;
break;
}
spin_unlock(&ce->guc_state.lock);
if (found) {
intel_engine_set_hung_context(engine, ce);
/* Can only cope with one hang at a time... */
intel_context_put(ce);
xa_lock(&guc->context_lookup);
goto done;
}
next:
intel_context_put(ce);
xa_lock(&guc->context_lookup);
}
done:
xa_unlock_irqrestore(&guc->context_lookup, flags);
}
void intel_guc_dump_active_requests(struct intel_engine_cs *engine,
struct i915_request *hung_rq,
struct drm_printer *m)
{
struct intel_guc *guc = gt_to_guc(engine->gt);
struct intel_context *ce;
unsigned long index;
unsigned long flags;
/* Reset called during driver load? GuC not yet initialised! */
if (unlikely(!guc_submission_initialized(guc)))
return;
xa_lock_irqsave(&guc->context_lookup, flags);
xa_for_each(&guc->context_lookup, index, ce) {
if (!kref_get_unless_zero(&ce->ref))
continue;
xa_unlock(&guc->context_lookup);
if (!intel_context_is_pinned(ce))
goto next;
if (intel_engine_is_virtual(ce->engine)) {
if (!(ce->engine->mask & engine->mask))
goto next;
} else {
if (ce->engine != engine)
goto next;
}
spin_lock(&ce->guc_state.lock);
intel_engine_dump_active_requests(&ce->guc_state.requests,
hung_rq, m);
spin_unlock(&ce->guc_state.lock);
next:
intel_context_put(ce);
xa_lock(&guc->context_lookup);
}
xa_unlock_irqrestore(&guc->context_lookup, flags);
}
void intel_guc_submission_print_info(struct intel_guc *guc,
struct drm_printer *p)
{
struct i915_sched_engine *sched_engine = guc->sched_engine;
struct rb_node *rb;
unsigned long flags;
if (!sched_engine)
return;
drm_printf(p, "GuC Submission API Version: %d.%d.%d\n",
guc->submission_version.major, guc->submission_version.minor,
guc->submission_version.patch);
drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n",
atomic_read(&guc->outstanding_submission_g2h));
drm_printf(p, "GuC tasklet count: %u\n",
atomic_read(&sched_engine->tasklet.count));
spin_lock_irqsave(&sched_engine->lock, flags);
drm_printf(p, "Requests in GuC submit tasklet:\n");
for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {
struct i915_priolist *pl = to_priolist(rb);
struct i915_request *rq;
priolist_for_each_request(rq, pl)
drm_printf(p, "guc_id=%u, seqno=%llu\n",
rq->context->guc_id.id,
rq->fence.seqno);
}
spin_unlock_irqrestore(&sched_engine->lock, flags);
drm_printf(p, "\n");
}
static inline void guc_log_context_priority(struct drm_printer *p,
struct intel_context *ce)
{
int i;
drm_printf(p, "\t\tPriority: %d\n", ce->guc_state.prio);
drm_printf(p, "\t\tNumber Requests (lower index == higher priority)\n");
for (i = GUC_CLIENT_PRIORITY_KMD_HIGH;
i < GUC_CLIENT_PRIORITY_NUM; ++i) {
drm_printf(p, "\t\tNumber requests in priority band[%d]: %d\n",
i, ce->guc_state.prio_count[i]);
}
drm_printf(p, "\n");
}
static inline void guc_log_context(struct drm_printer *p,
struct intel_context *ce)
{
drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id.id);
drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca);
drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n",
ce->ring->head,
ce->lrc_reg_state[CTX_RING_HEAD]);
drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n",
ce->ring->tail,
ce->lrc_reg_state[CTX_RING_TAIL]);
drm_printf(p, "\t\tContext Pin Count: %u\n",
atomic_read(&ce->pin_count));
drm_printf(p, "\t\tGuC ID Ref Count: %u\n",
atomic_read(&ce->guc_id.ref));
drm_printf(p, "\t\tSchedule State: 0x%x\n",
ce->guc_state.sched_state);
}
void intel_guc_submission_print_context_info(struct intel_guc *guc,
struct drm_printer *p)
{
struct intel_context *ce;
unsigned long index;
unsigned long flags;
xa_lock_irqsave(&guc->context_lookup, flags);
xa_for_each(&guc->context_lookup, index, ce) {
GEM_BUG_ON(intel_context_is_child(ce));
guc_log_context(p, ce);
guc_log_context_priority(p, ce);
if (intel_context_is_parent(ce)) {
struct intel_context *child;
drm_printf(p, "\t\tNumber children: %u\n",
ce->parallel.number_children);
if (ce->parallel.guc.wq_status) {
drm_printf(p, "\t\tWQI Head: %u\n",
READ_ONCE(*ce->parallel.guc.wq_head));
drm_printf(p, "\t\tWQI Tail: %u\n",
READ_ONCE(*ce->parallel.guc.wq_tail));
drm_printf(p, "\t\tWQI Status: %u\n",
READ_ONCE(*ce->parallel.guc.wq_status));
}
if (ce->engine->emit_bb_start ==
emit_bb_start_parent_no_preempt_mid_batch) {
u8 i;
drm_printf(p, "\t\tChildren Go: %u\n",
get_children_go_value(ce));
for (i = 0; i < ce->parallel.number_children; ++i)
drm_printf(p, "\t\tChildren Join: %u\n",
get_children_join_value(ce, i));
}
for_each_child(ce, child)
guc_log_context(p, child);
}
}
xa_unlock_irqrestore(&guc->context_lookup, flags);
}
static inline u32 get_children_go_addr(struct intel_context *ce)
{
GEM_BUG_ON(!intel_context_is_parent(ce));
return i915_ggtt_offset(ce->state) +
__get_parent_scratch_offset(ce) +
offsetof(struct parent_scratch, go.semaphore);
}
static inline u32 get_children_join_addr(struct intel_context *ce,
u8 child_index)
{
GEM_BUG_ON(!intel_context_is_parent(ce));
return i915_ggtt_offset(ce->state) +
__get_parent_scratch_offset(ce) +
offsetof(struct parent_scratch, join[child_index].semaphore);
}
#define PARENT_GO_BB 1
#define PARENT_GO_FINI_BREADCRUMB 0
#define CHILD_GO_BB 1
#define CHILD_GO_FINI_BREADCRUMB 0
static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
u64 offset, u32 len,
const unsigned int flags)
{
struct intel_context *ce = rq->context;
u32 *cs;
u8 i;
GEM_BUG_ON(!intel_context_is_parent(ce));
cs = intel_ring_begin(rq, 10 + 4 * ce->parallel.number_children);
if (IS_ERR(cs))
return PTR_ERR(cs);
/* Wait on children */
for (i = 0; i < ce->parallel.number_children; ++i) {
*cs++ = (MI_SEMAPHORE_WAIT |
MI_SEMAPHORE_GLOBAL_GTT |
MI_SEMAPHORE_POLL |
MI_SEMAPHORE_SAD_EQ_SDD);
*cs++ = PARENT_GO_BB;
*cs++ = get_children_join_addr(ce, i);
*cs++ = 0;
}
/* Turn off preemption */
*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
*cs++ = MI_NOOP;
/* Tell children go */
cs = gen8_emit_ggtt_write(cs,
CHILD_GO_BB,
get_children_go_addr(ce),
0);
/* Jump to batch */
*cs++ = MI_BATCH_BUFFER_START_GEN8 |
(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
*cs++ = lower_32_bits(offset);
*cs++ = upper_32_bits(offset);
*cs++ = MI_NOOP;
intel_ring_advance(rq, cs);
return 0;
}
static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
u64 offset, u32 len,
const unsigned int flags)
{
struct intel_context *ce = rq->context;
struct intel_context *parent = intel_context_to_parent(ce);
u32 *cs;
GEM_BUG_ON(!intel_context_is_child(ce));
cs = intel_ring_begin(rq, 12);
if (IS_ERR(cs))
return PTR_ERR(cs);
/* Signal parent */
cs = gen8_emit_ggtt_write(cs,
PARENT_GO_BB,
get_children_join_addr(parent,
ce->parallel.child_index),
0);
/* Wait on parent for go */
*cs++ = (MI_SEMAPHORE_WAIT |
MI_SEMAPHORE_GLOBAL_GTT |
MI_SEMAPHORE_POLL |
MI_SEMAPHORE_SAD_EQ_SDD);
*cs++ = CHILD_GO_BB;
*cs++ = get_children_go_addr(parent);
*cs++ = 0;
/* Turn off preemption */
*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
/* Jump to batch */
*cs++ = MI_BATCH_BUFFER_START_GEN8 |
(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
*cs++ = lower_32_bits(offset);
*cs++ = upper_32_bits(offset);
intel_ring_advance(rq, cs);
return 0;
}
static u32 *
__emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
u32 *cs)
{
struct intel_context *ce = rq->context;
u8 i;
GEM_BUG_ON(!intel_context_is_parent(ce));
/* Wait on children */
for (i = 0; i < ce->parallel.number_children; ++i) {
*cs++ = (MI_SEMAPHORE_WAIT |
MI_SEMAPHORE_GLOBAL_GTT |
MI_SEMAPHORE_POLL |
MI_SEMAPHORE_SAD_EQ_SDD);
*cs++ = PARENT_GO_FINI_BREADCRUMB;
*cs++ = get_children_join_addr(ce, i);
*cs++ = 0;
}
/* Turn on preemption */
*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
*cs++ = MI_NOOP;
/* Tell children go */
cs = gen8_emit_ggtt_write(cs,
CHILD_GO_FINI_BREADCRUMB,
get_children_go_addr(ce),
0);
return cs;
}
/*
* If this true, a submission of multi-lrc requests had an error and the
* requests need to be skipped. The front end (execuf IOCTL) should've called
* i915_request_skip which squashes the BB but we still need to emit the fini
* breadrcrumbs seqno write. At this point we don't know how many of the
* requests in the multi-lrc submission were generated so we can't do the
* handshake between the parent and children (e.g. if 4 requests should be
* generated but 2nd hit an error only 1 would be seen by the GuC backend).
* Simply skip the handshake, but still emit the breadcrumbd seqno, if an error
* has occurred on any of the requests in submission / relationship.
*/
static inline bool skip_handshake(struct i915_request *rq)
{
return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags);
}
#define NON_SKIP_LEN 6
static u32 *
emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
u32 *cs)
{
struct intel_context *ce = rq->context;
__maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;
__maybe_unused u32 *start_fini_breadcrumb_cs = cs;
GEM_BUG_ON(!intel_context_is_parent(ce));
if (unlikely(skip_handshake(rq))) {
/*
* NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch,
* the NON_SKIP_LEN comes from the length of the emits below.
*/
memset(cs, 0, sizeof(u32) *
(ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));
cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
} else {
cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs);
}
/* Emit fini breadcrumb */
before_fini_breadcrumb_user_interrupt_cs = cs;
cs = gen8_emit_ggtt_write(cs,
rq->fence.seqno,
i915_request_active_timeline(rq)->hwsp_offset,
0);
/* User interrupt */
*cs++ = MI_USER_INTERRUPT;
*cs++ = MI_NOOP;
/* Ensure our math for skip + emit is correct */
GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=
cs);
GEM_BUG_ON(start_fini_breadcrumb_cs +
ce->engine->emit_fini_breadcrumb_dw != cs);
rq->tail = intel_ring_offset(rq, cs);
return cs;
}
static u32 *
__emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
u32 *cs)
{
struct intel_context *ce = rq->context;
struct intel_context *parent = intel_context_to_parent(ce);
GEM_BUG_ON(!intel_context_is_child(ce));
/* Turn on preemption */
*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
*cs++ = MI_NOOP;
/* Signal parent */
cs = gen8_emit_ggtt_write(cs,
PARENT_GO_FINI_BREADCRUMB,
get_children_join_addr(parent,
ce->parallel.child_index),
0);
/* Wait parent on for go */
*cs++ = (MI_SEMAPHORE_WAIT |
MI_SEMAPHORE_GLOBAL_GTT |
MI_SEMAPHORE_POLL |
MI_SEMAPHORE_SAD_EQ_SDD);
*cs++ = CHILD_GO_FINI_BREADCRUMB;
*cs++ = get_children_go_addr(parent);
*cs++ = 0;
return cs;
}
static u32 *
emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
u32 *cs)
{
struct intel_context *ce = rq->context;
__maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;
__maybe_unused u32 *start_fini_breadcrumb_cs = cs;
GEM_BUG_ON(!intel_context_is_child(ce));
if (unlikely(skip_handshake(rq))) {
/*
* NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch,
* the NON_SKIP_LEN comes from the length of the emits below.
*/
memset(cs, 0, sizeof(u32) *
(ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));
cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
} else {
cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs);
}
/* Emit fini breadcrumb */
before_fini_breadcrumb_user_interrupt_cs = cs;
cs = gen8_emit_ggtt_write(cs,
rq->fence.seqno,
i915_request_active_timeline(rq)->hwsp_offset,
0);
/* User interrupt */
*cs++ = MI_USER_INTERRUPT;
*cs++ = MI_NOOP;
/* Ensure our math for skip + emit is correct */
GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=
cs);
GEM_BUG_ON(start_fini_breadcrumb_cs +
ce->engine->emit_fini_breadcrumb_dw != cs);
rq->tail = intel_ring_offset(rq, cs);
return cs;
}
#undef NON_SKIP_LEN
static struct intel_context *
guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
unsigned long flags)
{
struct guc_virtual_engine *ve;
struct intel_guc *guc;
unsigned int n;
int err;
ve = kzalloc(sizeof(*ve), GFP_KERNEL);
if (!ve)
return ERR_PTR(-ENOMEM);
guc = gt_to_guc(siblings[0]->gt);
ve->base.i915 = siblings[0]->i915;
ve->base.gt = siblings[0]->gt;
ve->base.uncore = siblings[0]->uncore;
ve->base.id = -1;
ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
ve->base.saturated = ALL_ENGINES;
snprintf(ve->base.name, sizeof(ve->base.name), "virtual");
ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine);
ve->base.cops = &virtual_guc_context_ops;
ve->base.request_alloc = guc_request_alloc;
ve->base.bump_serial = virtual_guc_bump_serial;
ve->base.submit_request = guc_submit_request;
ve->base.flags = I915_ENGINE_IS_VIRTUAL;
BUILD_BUG_ON(ilog2(VIRTUAL_ENGINES) < I915_NUM_ENGINES);
ve->base.mask = VIRTUAL_ENGINES;
intel_context_init(&ve->context, &ve->base);
for (n = 0; n < count; n++) {
struct intel_engine_cs *sibling = siblings[n];
GEM_BUG_ON(!is_power_of_2(sibling->mask));
if (sibling->mask & ve->base.mask) {
guc_dbg(guc, "duplicate %s entry in load balancer\n",
sibling->name);
err = -EINVAL;
goto err_put;
}
ve->base.mask |= sibling->mask;
ve->base.logical_mask |= sibling->logical_mask;
if (n != 0 && ve->base.class != sibling->class) {
guc_dbg(guc, "invalid mixing of engine class, sibling %d, already %d\n",
sibling->class, ve->base.class);
err = -EINVAL;
goto err_put;
} else if (n == 0) {
ve->base.class = sibling->class;
ve->base.uabi_class = sibling->uabi_class;
snprintf(ve->base.name, sizeof(ve->base.name),
"v%dx%d", ve->base.class, count);
ve->base.context_size = sibling->context_size;
ve->base.add_active_request =
sibling->add_active_request;
ve->base.remove_active_request =
sibling->remove_active_request;
ve->base.emit_bb_start = sibling->emit_bb_start;
ve->base.emit_flush = sibling->emit_flush;
ve->base.emit_init_breadcrumb =
sibling->emit_init_breadcrumb;
ve->base.emit_fini_breadcrumb =
sibling->emit_fini_breadcrumb;
ve->base.emit_fini_breadcrumb_dw =
sibling->emit_fini_breadcrumb_dw;
ve->base.breadcrumbs =
intel_breadcrumbs_get(sibling->breadcrumbs);
ve->base.flags |= sibling->flags;
ve->base.props.timeslice_duration_ms =
sibling->props.timeslice_duration_ms;
ve->base.props.preempt_timeout_ms =
sibling->props.preempt_timeout_ms;
}
}
return &ve->context;
err_put:
intel_context_put(&ve->context);
return ERR_PTR(err);
}
bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve)
{
struct intel_engine_cs *engine;
intel_engine_mask_t tmp, mask = ve->mask;
for_each_engine_masked(engine, ve->gt, mask, tmp)
if (READ_ONCE(engine->props.heartbeat_interval_ms))
return true;
return false;
}
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftest_guc.c"
#include "selftest_guc_multi_lrc.c"
#include "selftest_guc_hangcheck.c"
#endif
Codebase Browser
linux