// SPDX-License-Identifier: GPL-2.0 or MIT
/* Copyright 2023 Collabora ltd. */
#include <drm/drm_drv.h>
#include <drm/drm_exec.h>
#include <drm/drm_gem_shmem_helper.h>
#include <drm/drm_managed.h>
#include <drm/gpu_scheduler.h>
#include <drm/panthor_drm.h>
#include <linux/build_bug.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/dma-resv.h>
#include <linux/firmware.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/iosys-map.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include "panthor_devfreq.h"
#include "panthor_device.h"
#include "panthor_fw.h"
#include "panthor_gem.h"
#include "panthor_gpu.h"
#include "panthor_heap.h"
#include "panthor_mmu.h"
#include "panthor_regs.h"
#include "panthor_sched.h"
/**
* DOC: Scheduler
*
* Mali CSF hardware adopts a firmware-assisted scheduling model, where
* the firmware takes care of scheduling aspects, to some extent.
*
* The scheduling happens at the scheduling group level, each group
* contains 1 to N queues (N is FW/hardware dependent, and exposed
* through the firmware interface). Each queue is assigned a command
* stream ring buffer, which serves as a way to get jobs submitted to
* the GPU, among other things.
*
* The firmware can schedule a maximum of M groups (M is FW/hardware
* dependent, and exposed through the firmware interface). Passed
* this maximum number of groups, the kernel must take care of
* rotating the groups passed to the firmware so every group gets
* a chance to have his queues scheduled for execution.
*
* The current implementation only supports with kernel-mode queues.
* In other terms, userspace doesn't have access to the ring-buffer.
* Instead, userspace passes indirect command stream buffers that are
* called from the queue ring-buffer by the kernel using a pre-defined
* sequence of command stream instructions to ensure the userspace driver
* always gets consistent results (cache maintenance,
* synchronization, ...).
*
* We rely on the drm_gpu_scheduler framework to deal with job
* dependencies and submission. As any other driver dealing with a
* FW-scheduler, we use the 1:1 entity:scheduler mode, such that each
* entity has its own job scheduler. When a job is ready to be executed
* (all its dependencies are met), it is pushed to the appropriate
* queue ring-buffer, and the group is scheduled for execution if it
* wasn't already active.
*
* Kernel-side group scheduling is timeslice-based. When we have less
* groups than there are slots, the periodic tick is disabled and we
* just let the FW schedule the active groups. When there are more
* groups than slots, we let each group a chance to execute stuff for
* a given amount of time, and then re-evaluate and pick new groups
* to schedule. The group selection algorithm is based on
* priority+round-robin.
*
* Even though user-mode queues is out of the scope right now, the
* current design takes them into account by avoiding any guess on the
* group/queue state that would be based on information we wouldn't have
* if userspace was in charge of the ring-buffer. That's also one of the
* reason we don't do 'cooperative' scheduling (encoding FW group slot
* reservation as dma_fence that would be returned from the
* drm_gpu_scheduler::prepare_job() hook, and treating group rotation as
* a queue of waiters, ordered by job submission order). This approach
* would work for kernel-mode queues, but would make user-mode queues a
* lot more complicated to retrofit.
*/
#define JOB_TIMEOUT_MS 5000
#define MIN_CS_PER_CSG 8
#define MIN_CSGS 3
#define MAX_CSG_PRIO 0xf
struct panthor_group;
/**
* struct panthor_csg_slot - Command stream group slot
*
* This represents a FW slot for a scheduling group.
*/
struct panthor_csg_slot {
/** @group: Scheduling group bound to this slot. */
struct panthor_group *group;
/** @priority: Group priority. */
u8 priority;
/**
* @idle: True if the group bound to this slot is idle.
*
* A group is idle when it has nothing waiting for execution on
* all its queues, or when queues are blocked waiting for something
* to happen (synchronization object).
*/
bool idle;
};
/**
* enum panthor_csg_priority - Group priority
*/
enum panthor_csg_priority {
/** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */
PANTHOR_CSG_PRIORITY_LOW = 0,
/** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */
PANTHOR_CSG_PRIORITY_MEDIUM,
/** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */
PANTHOR_CSG_PRIORITY_HIGH,
/**
* @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.
*
* Real-time priority allows one to preempt scheduling of other
* non-real-time groups. When such a group becomes executable,
* it will evict the group with the lowest non-rt priority if
* there's no free group slot available.
*
* Currently not exposed to userspace.
*/
PANTHOR_CSG_PRIORITY_RT,
/** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */
PANTHOR_CSG_PRIORITY_COUNT,
};
/**
* struct panthor_scheduler - Object used to manage the scheduler
*/
struct panthor_scheduler {
/** @ptdev: Device. */
struct panthor_device *ptdev;
/**
* @wq: Workqueue used by our internal scheduler logic and
* drm_gpu_scheduler.
*
* Used for the scheduler tick, group update or other kind of FW
* event processing that can't be handled in the threaded interrupt
* path. Also passed to the drm_gpu_scheduler instances embedded
* in panthor_queue.
*/
struct workqueue_struct *wq;
/**
* @heap_alloc_wq: Workqueue used to schedule tiler_oom works.
*
* We have a queue dedicated to heap chunk allocation works to avoid
* blocking the rest of the scheduler if the allocation tries to
* reclaim memory.
*/
struct workqueue_struct *heap_alloc_wq;
/** @tick_work: Work executed on a scheduling tick. */
struct delayed_work tick_work;
/**
* @sync_upd_work: Work used to process synchronization object updates.
*
* We use this work to unblock queues/groups that were waiting on a
* synchronization object.
*/
struct work_struct sync_upd_work;
/**
* @fw_events_work: Work used to process FW events outside the interrupt path.
*
* Even if the interrupt is threaded, we need any event processing
* that require taking the panthor_scheduler::lock to be processed
* outside the interrupt path so we don't block the tick logic when
* it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the
* event processing requires taking this lock, we just delegate all
* FW event processing to the scheduler workqueue.
*/
struct work_struct fw_events_work;
/**
* @fw_events: Bitmask encoding pending FW events.
*/
atomic_t fw_events;
/**
* @resched_target: When the next tick should occur.
*
* Expressed in jiffies.
*/
u64 resched_target;
/**
* @last_tick: When the last tick occurred.
*
* Expressed in jiffies.
*/
u64 last_tick;
/** @tick_period: Tick period in jiffies. */
u64 tick_period;
/**
* @lock: Lock protecting access to all the scheduler fields.
*
* Should be taken in the tick work, the irq handler, and anywhere the @groups
* fields are touched.
*/
struct mutex lock;
/** @groups: Various lists used to classify groups. */
struct {
/**
* @runnable: Runnable group lists.
*
* When a group has queues that want to execute something,
* its panthor_group::run_node should be inserted here.
*
* One list per-priority.
*/
struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT];
/**
* @idle: Idle group lists.
*
* When all queues of a group are idle (either because they
* have nothing to execute, or because they are blocked), the
* panthor_group::run_node field should be inserted here.
*
* One list per-priority.
*/
struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT];
/**
* @waiting: List of groups whose queues are blocked on a
* synchronization object.
*
* Insert panthor_group::wait_node here when a group is waiting
* for synchronization objects to be signaled.
*
* This list is evaluated in the @sync_upd_work work.
*/
struct list_head waiting;
} groups;
/**
* @csg_slots: FW command stream group slots.
*/
struct panthor_csg_slot csg_slots[MAX_CSGS];
/** @csg_slot_count: Number of command stream group slots exposed by the FW. */
u32 csg_slot_count;
/** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */
u32 cs_slot_count;
/** @as_slot_count: Number of address space slots supported by the MMU. */
u32 as_slot_count;
/** @used_csg_slot_count: Number of command stream group slot currently used. */
u32 used_csg_slot_count;
/** @sb_slot_count: Number of scoreboard slots. */
u32 sb_slot_count;
/**
* @might_have_idle_groups: True if an active group might have become idle.
*
* This will force a tick, so other runnable groups can be scheduled if one
* or more active groups became idle.
*/
bool might_have_idle_groups;
/** @pm: Power management related fields. */
struct {
/** @has_ref: True if the scheduler owns a runtime PM reference. */
bool has_ref;
} pm;
/** @reset: Reset related fields. */
struct {
/** @lock: Lock protecting the other reset fields. */
struct mutex lock;
/**
* @in_progress: True if a reset is in progress.
*
* Set to true in panthor_sched_pre_reset() and back to false in
* panthor_sched_post_reset().
*/
atomic_t in_progress;
/**
* @stopped_groups: List containing all groups that were stopped
* before a reset.
*
* Insert panthor_group::run_node in the pre_reset path.
*/
struct list_head stopped_groups;
} reset;
};
/**
* struct panthor_syncobj_32b - 32-bit FW synchronization object
*/
struct panthor_syncobj_32b {
/** @seqno: Sequence number. */
u32 seqno;
/**
* @status: Status.
*
* Not zero on failure.
*/
u32 status;
};
/**
* struct panthor_syncobj_64b - 64-bit FW synchronization object
*/
struct panthor_syncobj_64b {
/** @seqno: Sequence number. */
u64 seqno;
/**
* @status: Status.
*
* Not zero on failure.
*/
u32 status;
/** @pad: MBZ. */
u32 pad;
};
/**
* struct panthor_queue - Execution queue
*/
struct panthor_queue {
/** @scheduler: DRM scheduler used for this queue. */
struct drm_gpu_scheduler scheduler;
/** @entity: DRM scheduling entity used for this queue. */
struct drm_sched_entity entity;
/**
* @remaining_time: Time remaining before the job timeout expires.
*
* The job timeout is suspended when the queue is not scheduled by the
* FW. Every time we suspend the timer, we need to save the remaining
* time so we can restore it later on.
*/
unsigned long remaining_time;
/** @timeout_suspended: True if the job timeout was suspended. */
bool timeout_suspended;
/**
* @doorbell_id: Doorbell assigned to this queue.
*
* Right now, all groups share the same doorbell, and the doorbell ID
* is assigned to group_slot + 1 when the group is assigned a slot. But
* we might decide to provide fine grained doorbell assignment at some
* point, so don't have to wake up all queues in a group every time one
* of them is updated.
*/
u8 doorbell_id;
/**
* @priority: Priority of the queue inside the group.
*
* Must be less than 16 (Only 4 bits available).
*/
u8 priority;
#define CSF_MAX_QUEUE_PRIO GENMASK(3, 0)
/** @ringbuf: Command stream ring-buffer. */
struct panthor_kernel_bo *ringbuf;
/** @iface: Firmware interface. */
struct {
/** @mem: FW memory allocated for this interface. */
struct panthor_kernel_bo *mem;
/** @input: Input interface. */
struct panthor_fw_ringbuf_input_iface *input;
/** @output: Output interface. */
const struct panthor_fw_ringbuf_output_iface *output;
/** @input_fw_va: FW virtual address of the input interface buffer. */
u32 input_fw_va;
/** @output_fw_va: FW virtual address of the output interface buffer. */
u32 output_fw_va;
} iface;
/**
* @syncwait: Stores information about the synchronization object this
* queue is waiting on.
*/
struct {
/** @gpu_va: GPU address of the synchronization object. */
u64 gpu_va;
/** @ref: Reference value to compare against. */
u64 ref;
/** @gt: True if this is a greater-than test. */
bool gt;
/** @sync64: True if this is a 64-bit sync object. */
bool sync64;
/** @bo: Buffer object holding the synchronization object. */
struct drm_gem_object *obj;
/** @offset: Offset of the synchronization object inside @bo. */
u64 offset;
/**
* @kmap: Kernel mapping of the buffer object holding the
* synchronization object.
*/
void *kmap;
} syncwait;
/** @fence_ctx: Fence context fields. */
struct {
/** @lock: Used to protect access to all fences allocated by this context. */
spinlock_t lock;
/**
* @id: Fence context ID.
*
* Allocated with dma_fence_context_alloc().
*/
u64 id;
/** @seqno: Sequence number of the last initialized fence. */
atomic64_t seqno;
/**
* @last_fence: Fence of the last submitted job.
*
* We return this fence when we get an empty command stream.
* This way, we are guaranteed that all earlier jobs have completed
* when drm_sched_job::s_fence::finished without having to feed
* the CS ring buffer with a dummy job that only signals the fence.
*/
struct dma_fence *last_fence;
/**
* @in_flight_jobs: List containing all in-flight jobs.
*
* Used to keep track and signal panthor_job::done_fence when the
* synchronization object attached to the queue is signaled.
*/
struct list_head in_flight_jobs;
} fence_ctx;
};
/**
* enum panthor_group_state - Scheduling group state.
*/
enum panthor_group_state {
/** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */
PANTHOR_CS_GROUP_CREATED,
/** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */
PANTHOR_CS_GROUP_ACTIVE,
/**
* @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is
* inactive/suspended right now.
*/
PANTHOR_CS_GROUP_SUSPENDED,
/**
* @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.
*
* Can no longer be scheduled. The only allowed action is a destruction.
*/
PANTHOR_CS_GROUP_TERMINATED,
/**
* @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.
*
* The FW returned an inconsistent state. The group is flagged unusable
* and can no longer be scheduled. The only allowed action is a
* destruction.
*
* When that happens, we also schedule a FW reset, to start from a fresh
* state.
*/
PANTHOR_CS_GROUP_UNKNOWN_STATE,
};
/**
* struct panthor_group - Scheduling group object
*/
struct panthor_group {
/** @refcount: Reference count */
struct kref refcount;
/** @ptdev: Device. */
struct panthor_device *ptdev;
/** @vm: VM bound to the group. */
struct panthor_vm *vm;
/** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */
u64 compute_core_mask;
/** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */
u64 fragment_core_mask;
/** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */
u64 tiler_core_mask;
/** @max_compute_cores: Maximum number of shader cores used for compute jobs. */
u8 max_compute_cores;
/** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */
u8 max_fragment_cores;
/** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */
u8 max_tiler_cores;
/** @priority: Group priority (check panthor_csg_priority). */
u8 priority;
/** @blocked_queues: Bitmask reflecting the blocked queues. */
u32 blocked_queues;
/** @idle_queues: Bitmask reflecting the idle queues. */
u32 idle_queues;
/** @fatal_lock: Lock used to protect access to fatal fields. */
spinlock_t fatal_lock;
/** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */
u32 fatal_queues;
/** @tiler_oom: Mask of queues that have a tiler OOM event to process. */
atomic_t tiler_oom;
/** @queue_count: Number of queues in this group. */
u32 queue_count;
/** @queues: Queues owned by this group. */
struct panthor_queue *queues[MAX_CS_PER_CSG];
/**
* @csg_id: ID of the FW group slot.
*
* -1 when the group is not scheduled/active.
*/
int csg_id;
/**
* @destroyed: True when the group has been destroyed.
*
* If a group is destroyed it becomes useless: no further jobs can be submitted
* to its queues. We simply wait for all references to be dropped so we can
* release the group object.
*/
bool destroyed;
/**
* @timedout: True when a timeout occurred on any of the queues owned by
* this group.
*
* Timeouts can be reported by drm_sched or by the FW. In any case, any
* timeout situation is unrecoverable, and the group becomes useless.
* We simply wait for all references to be dropped so we can release the
* group object.
*/
bool timedout;
/**
* @syncobjs: Pool of per-queue synchronization objects.
*
* One sync object per queue. The position of the sync object is
* determined by the queue index.
*/
struct panthor_kernel_bo *syncobjs;
/** @state: Group state. */
enum panthor_group_state state;
/**
* @suspend_buf: Suspend buffer.
*
* Stores the state of the group and its queues when a group is suspended.
* Used at resume time to restore the group in its previous state.
*
* The size of the suspend buffer is exposed through the FW interface.
*/
struct panthor_kernel_bo *suspend_buf;
/**
* @protm_suspend_buf: Protection mode suspend buffer.
*
* Stores the state of the group and its queues when a group that's in
* protection mode is suspended.
*
* Used at resume time to restore the group in its previous state.
*
* The size of the protection mode suspend buffer is exposed through the
* FW interface.
*/
struct panthor_kernel_bo *protm_suspend_buf;
/** @sync_upd_work: Work used to check/signal job fences. */
struct work_struct sync_upd_work;
/** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
struct work_struct tiler_oom_work;
/** @term_work: Work used to finish the group termination procedure. */
struct work_struct term_work;
/**
* @release_work: Work used to release group resources.
*
* We need to postpone the group release to avoid a deadlock when
* the last ref is released in the tick work.
*/
struct work_struct release_work;
/**
* @run_node: Node used to insert the group in the
* panthor_group::groups::{runnable,idle} and
* panthor_group::reset.stopped_groups lists.
*/
struct list_head run_node;
/**
* @wait_node: Node used to insert the group in the
* panthor_group::groups::waiting list.
*/
struct list_head wait_node;
};
/**
* group_queue_work() - Queue a group work
* @group: Group to queue the work for.
* @wname: Work name.
*
* Grabs a ref and queue a work item to the scheduler workqueue. If
* the work was already queued, we release the reference we grabbed.
*
* Work callbacks must release the reference we grabbed here.
*/
#define group_queue_work(group, wname) \
do { \
group_get(group); \
if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \
group_put(group); \
} while (0)
/**
* sched_queue_work() - Queue a scheduler work.
* @sched: Scheduler object.
* @wname: Work name.
*
* Conditionally queues a scheduler work if no reset is pending/in-progress.
*/
#define sched_queue_work(sched, wname) \
do { \
if (!atomic_read(&(sched)->reset.in_progress) && \
!panthor_device_reset_is_pending((sched)->ptdev)) \
queue_work((sched)->wq, &(sched)->wname ## _work); \
} while (0)
/**
* sched_queue_delayed_work() - Queue a scheduler delayed work.
* @sched: Scheduler object.
* @wname: Work name.
* @delay: Work delay in jiffies.
*
* Conditionally queues a scheduler delayed work if no reset is
* pending/in-progress.
*/
#define sched_queue_delayed_work(sched, wname, delay) \
do { \
if (!atomic_read(&sched->reset.in_progress) && \
!panthor_device_reset_is_pending((sched)->ptdev)) \
mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \
} while (0)
/*
* We currently set the maximum of groups per file to an arbitrary low value.
* But this can be updated if we need more.
*/
#define MAX_GROUPS_PER_POOL 128
/**
* struct panthor_group_pool - Group pool
*
* Each file get assigned a group pool.
*/
struct panthor_group_pool {
/** @xa: Xarray used to manage group handles. */
struct xarray xa;
};
/**
* struct panthor_job - Used to manage GPU job
*/
struct panthor_job {
/** @base: Inherit from drm_sched_job. */
struct drm_sched_job base;
/** @refcount: Reference count. */
struct kref refcount;
/** @group: Group of the queue this job will be pushed to. */
struct panthor_group *group;
/** @queue_idx: Index of the queue inside @group. */
u32 queue_idx;
/** @call_info: Information about the userspace command stream call. */
struct {
/** @start: GPU address of the userspace command stream. */
u64 start;
/** @size: Size of the userspace command stream. */
u32 size;
/**
* @latest_flush: Flush ID at the time the userspace command
* stream was built.
*
* Needed for the flush reduction mechanism.
*/
u32 latest_flush;
} call_info;
/** @ringbuf: Position of this job is in the ring buffer. */
struct {
/** @start: Start offset. */
u64 start;
/** @end: End offset. */
u64 end;
} ringbuf;
/**
* @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs
* list.
*/
struct list_head node;
/** @done_fence: Fence signaled when the job is finished or cancelled. */
struct dma_fence *done_fence;
};
static void
panthor_queue_put_syncwait_obj(struct panthor_queue *queue)
{
if (queue->syncwait.kmap) {
struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap);
drm_gem_vunmap_unlocked(queue->syncwait.obj, &map);
queue->syncwait.kmap = NULL;
}
drm_gem_object_put(queue->syncwait.obj);
queue->syncwait.obj = NULL;
}
static void *
panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue)
{
struct panthor_device *ptdev = group->ptdev;
struct panthor_gem_object *bo;
struct iosys_map map;
int ret;
if (queue->syncwait.kmap)
return queue->syncwait.kmap + queue->syncwait.offset;
bo = panthor_vm_get_bo_for_va(group->vm,
queue->syncwait.gpu_va,
&queue->syncwait.offset);
if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))
goto err_put_syncwait_obj;
queue->syncwait.obj = &bo->base.base;
ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map);
if (drm_WARN_ON(&ptdev->base, ret))
goto err_put_syncwait_obj;
queue->syncwait.kmap = map.vaddr;
if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))
goto err_put_syncwait_obj;
return queue->syncwait.kmap + queue->syncwait.offset;
err_put_syncwait_obj:
panthor_queue_put_syncwait_obj(queue);
return NULL;
}
static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue)
{
if (IS_ERR_OR_NULL(queue))
return;
if (queue->entity.fence_context)
drm_sched_entity_destroy(&queue->entity);
if (queue->scheduler.ops)
drm_sched_fini(&queue->scheduler);
panthor_queue_put_syncwait_obj(queue);
panthor_kernel_bo_destroy(queue->ringbuf);
panthor_kernel_bo_destroy(queue->iface.mem);
/* Release the last_fence we were holding, if any. */
dma_fence_put(queue->fence_ctx.last_fence);
kfree(queue);
}
static void group_release_work(struct work_struct *work)
{
struct panthor_group *group = container_of(work,
struct panthor_group,
release_work);
u32 i;
for (i = 0; i < group->queue_count; i++)
group_free_queue(group, group->queues[i]);
panthor_kernel_bo_destroy(group->suspend_buf);
panthor_kernel_bo_destroy(group->protm_suspend_buf);
panthor_kernel_bo_destroy(group->syncobjs);
panthor_vm_put(group->vm);
kfree(group);
}
static void group_release(struct kref *kref)
{
struct panthor_group *group = container_of(kref,
struct panthor_group,
refcount);
struct panthor_device *ptdev = group->ptdev;
drm_WARN_ON(&ptdev->base, group->csg_id >= 0);
drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node));
drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node));
queue_work(panthor_cleanup_wq, &group->release_work);
}
static void group_put(struct panthor_group *group)
{
if (group)
kref_put(&group->refcount, group_release);
}
static struct panthor_group *
group_get(struct panthor_group *group)
{
if (group)
kref_get(&group->refcount);
return group;
}
/**
* group_bind_locked() - Bind a group to a group slot
* @group: Group.
* @csg_id: Slot.
*
* Return: 0 on success, a negative error code otherwise.
*/
static int
group_bind_locked(struct panthor_group *group, u32 csg_id)
{
struct panthor_device *ptdev = group->ptdev;
struct panthor_csg_slot *csg_slot;
int ret;
lockdep_assert_held(&ptdev->scheduler->lock);
if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||
ptdev->scheduler->csg_slots[csg_id].group))
return -EINVAL;
ret = panthor_vm_active(group->vm);
if (ret)
return ret;
csg_slot = &ptdev->scheduler->csg_slots[csg_id];
group_get(group);
group->csg_id = csg_id;
/* Dummy doorbell allocation: doorbell is assigned to the group and
* all queues use the same doorbell.
*
* TODO: Implement LRU-based doorbell assignment, so the most often
* updated queues get their own doorbell, thus avoiding useless checks
* on queues belonging to the same group that are rarely updated.
*/
for (u32 i = 0; i < group->queue_count; i++)
group->queues[i]->doorbell_id = csg_id + 1;
csg_slot->group = group;
return 0;
}
/**
* group_unbind_locked() - Unbind a group from a slot.
* @group: Group to unbind.
*
* Return: 0 on success, a negative error code otherwise.
*/
static int
group_unbind_locked(struct panthor_group *group)
{
struct panthor_device *ptdev = group->ptdev;
struct panthor_csg_slot *slot;
lockdep_assert_held(&ptdev->scheduler->lock);
if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))
return -EINVAL;
if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))
return -EINVAL;
slot = &ptdev->scheduler->csg_slots[group->csg_id];
panthor_vm_idle(group->vm);
group->csg_id = -1;
/* Tiler OOM events will be re-issued next time the group is scheduled. */
atomic_set(&group->tiler_oom, 0);
cancel_work(&group->tiler_oom_work);
for (u32 i = 0; i < group->queue_count; i++)
group->queues[i]->doorbell_id = -1;
slot->group = NULL;
group_put(group);
return 0;
}
/**
* cs_slot_prog_locked() - Program a queue slot
* @ptdev: Device.
* @csg_id: Group slot ID.
* @cs_id: Queue slot ID.
*
* Program a queue slot with the queue information so things can start being
* executed on this queue.
*
* The group slot must have a group bound to it already (group_bind_locked()).
*/
static void
cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
{
struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id];
struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
lockdep_assert_held(&ptdev->scheduler->lock);
queue->iface.input->extract = queue->iface.output->extract;
drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract);
cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf);
cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
cs_iface->input->ringbuf_input = queue->iface.input_fw_va;
cs_iface->input->ringbuf_output = queue->iface.output_fw_va;
cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) |
CS_CONFIG_DOORBELL(queue->doorbell_id);
cs_iface->input->ack_irq_mask = ~0;
panthor_fw_update_reqs(cs_iface, req,
CS_IDLE_SYNC_WAIT |
CS_IDLE_EMPTY |
CS_STATE_START |
CS_EXTRACT_EVENT,
CS_IDLE_SYNC_WAIT |
CS_IDLE_EMPTY |
CS_STATE_MASK |
CS_EXTRACT_EVENT);
if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) {
drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time);
queue->timeout_suspended = false;
}
}
/**
* cs_slot_reset_locked() - Reset a queue slot
* @ptdev: Device.
* @csg_id: Group slot.
* @cs_id: Queue slot.
*
* Change the queue slot state to STOP and suspend the queue timeout if
* the queue is not blocked.
*
* The group slot must have a group bound to it (group_bind_locked()).
*/
static int
cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
{
struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
struct panthor_queue *queue = group->queues[cs_id];
lockdep_assert_held(&ptdev->scheduler->lock);
panthor_fw_update_reqs(cs_iface, req,
CS_STATE_STOP,
CS_STATE_MASK);
/* If the queue is blocked, we want to keep the timeout running, so
* we can detect unbounded waits and kill the group when that happens.
*/
if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) {
queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
queue->timeout_suspended = true;
WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS));
}
return 0;
}
/**
* csg_slot_sync_priority_locked() - Synchronize the group slot priority
* @ptdev: Device.
* @csg_id: Group slot ID.
*
* Group slot priority update happens asynchronously. When we receive a
* %CSG_ENDPOINT_CONFIG, we know the update is effective, and can
* reflect it to our panthor_csg_slot object.
*/
static void
csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id)
{
struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
struct panthor_fw_csg_iface *csg_iface;
lockdep_assert_held(&ptdev->scheduler->lock);
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28;
}
/**
* cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority
* @ptdev: Device.
* @csg_id: Group slot.
* @cs_id: Queue slot.
*
* Queue state is updated on group suspend or STATUS_UPDATE event.
*/
static void
cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
{
struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
struct panthor_queue *queue = group->queues[cs_id];
struct panthor_fw_cs_iface *cs_iface =
panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id);
u32 status_wait_cond;
switch (cs_iface->output->status_blocked_reason) {
case CS_STATUS_BLOCKED_REASON_UNBLOCKED:
if (queue->iface.input->insert == queue->iface.output->extract &&
cs_iface->output->status_scoreboards == 0)
group->idle_queues |= BIT(cs_id);
break;
case CS_STATUS_BLOCKED_REASON_SYNC_WAIT:
if (list_empty(&group->wait_node)) {
list_move_tail(&group->wait_node,
&group->ptdev->scheduler->groups.waiting);
}
/* The queue is only blocked if there's no deferred operation
* pending, which can be checked through the scoreboard status.
*/
if (!cs_iface->output->status_scoreboards)
group->blocked_queues |= BIT(cs_id);
queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr;
queue->syncwait.ref = cs_iface->output->status_wait_sync_value;
status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK;
queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT;
if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) {
u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi;
queue->syncwait.sync64 = true;
queue->syncwait.ref |= sync_val_hi << 32;
} else {
queue->syncwait.sync64 = false;
}
break;
default:
/* Other reasons are not blocking. Consider the queue as runnable
* in those cases.
*/
break;
}
}
static void
csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id)
{
struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
struct panthor_group *group = csg_slot->group;
u32 i;
lockdep_assert_held(&ptdev->scheduler->lock);
group->idle_queues = 0;
group->blocked_queues = 0;
for (i = 0; i < group->queue_count; i++) {
if (group->queues[i])
cs_slot_sync_queue_state_locked(ptdev, csg_id, i);
}
}
static void
csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id)
{
struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
struct panthor_fw_csg_iface *csg_iface;
struct panthor_group *group;
enum panthor_group_state new_state, old_state;
u32 csg_state;
lockdep_assert_held(&ptdev->scheduler->lock);
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
group = csg_slot->group;
if (!group)
return;
old_state = group->state;
csg_state = csg_iface->output->ack & CSG_STATE_MASK;
switch (csg_state) {
case CSG_STATE_START:
case CSG_STATE_RESUME:
new_state = PANTHOR_CS_GROUP_ACTIVE;
break;
case CSG_STATE_TERMINATE:
new_state = PANTHOR_CS_GROUP_TERMINATED;
break;
case CSG_STATE_SUSPEND:
new_state = PANTHOR_CS_GROUP_SUSPENDED;
break;
default:
/* The unknown state might be caused by a FW state corruption,
* which means the group metadata can't be trusted anymore, and
* the SUSPEND operation might propagate the corruption to the
* suspend buffers. Flag the group state as unknown to make
* sure it's unusable after that point.
*/
drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)",
csg_id, csg_state);
new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE;
break;
}
if (old_state == new_state)
return;
/* The unknown state might be caused by a FW issue, reset the FW to
* take a fresh start.
*/
if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE)
panthor_device_schedule_reset(ptdev);
if (new_state == PANTHOR_CS_GROUP_SUSPENDED)
csg_slot_sync_queues_state_locked(ptdev, csg_id);
if (old_state == PANTHOR_CS_GROUP_ACTIVE) {
u32 i;
/* Reset the queue slots so we start from a clean
* state when starting/resuming a new group on this
* CSG slot. No wait needed here, and no ringbell
* either, since the CS slot will only be re-used
* on the next CSG start operation.
*/
for (i = 0; i < group->queue_count; i++) {
if (group->queues[i])
cs_slot_reset_locked(ptdev, csg_id, i);
}
}
group->state = new_state;
}
static int
csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority)
{
struct panthor_fw_csg_iface *csg_iface;
struct panthor_csg_slot *csg_slot;
struct panthor_group *group;
u32 queue_mask = 0, i;
lockdep_assert_held(&ptdev->scheduler->lock);
if (priority > MAX_CSG_PRIO)
return -EINVAL;
if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS))
return -EINVAL;
csg_slot = &ptdev->scheduler->csg_slots[csg_id];
group = csg_slot->group;
if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE)
return 0;
csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id);
for (i = 0; i < group->queue_count; i++) {
if (group->queues[i]) {
cs_slot_prog_locked(ptdev, csg_id, i);
queue_mask |= BIT(i);
}
}
csg_iface->input->allow_compute = group->compute_core_mask;
csg_iface->input->allow_fragment = group->fragment_core_mask;
csg_iface->input->allow_other = group->tiler_core_mask;
csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) |
CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) |
CSG_EP_REQ_TILER(group->max_tiler_cores) |
CSG_EP_REQ_PRIORITY(priority);
csg_iface->input->config = panthor_vm_as(group->vm);
if (group->suspend_buf)
csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf);
else
csg_iface->input->suspend_buf = 0;
if (group->protm_suspend_buf) {
csg_iface->input->protm_suspend_buf =
panthor_kernel_bo_gpuva(group->protm_suspend_buf);
} else {
csg_iface->input->protm_suspend_buf = 0;
}
csg_iface->input->ack_irq_mask = ~0;
panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask);
return 0;
}
static void
cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,
u32 csg_id, u32 cs_id)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
struct panthor_group *group = csg_slot->group;
struct panthor_fw_cs_iface *cs_iface;
u32 fatal;
u64 info;
lockdep_assert_held(&sched->lock);
cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
fatal = cs_iface->output->fatal;
info = cs_iface->output->fatal_info;
if (group)
group->fatal_queues |= BIT(cs_id);
if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) {
/* If this exception is unrecoverable, queue a reset, and make
* sure we stop scheduling groups until the reset has happened.
*/
panthor_device_schedule_reset(ptdev);
cancel_delayed_work(&sched->tick_work);
} else {
sched_queue_delayed_work(sched, tick, 0);
}
drm_warn(&ptdev->base,
"CSG slot %d CS slot: %d\n"
"CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"
"CS_FATAL.EXCEPTION_DATA: 0x%x\n"
"CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n",
csg_id, cs_id,
(unsigned int)CS_EXCEPTION_TYPE(fatal),
panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)),
(unsigned int)CS_EXCEPTION_DATA(fatal),
info);
}
static void
cs_slot_process_fault_event_locked(struct panthor_device *ptdev,
u32 csg_id, u32 cs_id)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
struct panthor_group *group = csg_slot->group;
struct panthor_queue *queue = group && cs_id < group->queue_count ?
group->queues[cs_id] : NULL;
struct panthor_fw_cs_iface *cs_iface;
u32 fault;
u64 info;
lockdep_assert_held(&sched->lock);
cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
fault = cs_iface->output->fault;
info = cs_iface->output->fault_info;
if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) {
u64 cs_extract = queue->iface.output->extract;
struct panthor_job *job;
spin_lock(&queue->fence_ctx.lock);
list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) {
if (cs_extract >= job->ringbuf.end)
continue;
if (cs_extract < job->ringbuf.start)
break;
dma_fence_set_error(job->done_fence, -EINVAL);
}
spin_unlock(&queue->fence_ctx.lock);
}
drm_warn(&ptdev->base,
"CSG slot %d CS slot: %d\n"
"CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
"CS_FAULT.EXCEPTION_DATA: 0x%x\n"
"CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",
csg_id, cs_id,
(unsigned int)CS_EXCEPTION_TYPE(fault),
panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)),
(unsigned int)CS_EXCEPTION_DATA(fault),
info);
}
static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id)
{
struct panthor_device *ptdev = group->ptdev;
struct panthor_scheduler *sched = ptdev->scheduler;
u32 renderpasses_in_flight, pending_frag_count;
struct panthor_heap_pool *heaps = NULL;
u64 heap_address, new_chunk_va = 0;
u32 vt_start, vt_end, frag_end;
int ret, csg_id;
mutex_lock(&sched->lock);
csg_id = group->csg_id;
if (csg_id >= 0) {
struct panthor_fw_cs_iface *cs_iface;
cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
heaps = panthor_vm_get_heap_pool(group->vm, false);
heap_address = cs_iface->output->heap_address;
vt_start = cs_iface->output->heap_vt_start;
vt_end = cs_iface->output->heap_vt_end;
frag_end = cs_iface->output->heap_frag_end;
renderpasses_in_flight = vt_start - frag_end;
pending_frag_count = vt_end - frag_end;
}
mutex_unlock(&sched->lock);
/* The group got scheduled out, we stop here. We will get a new tiler OOM event
* when it's scheduled again.
*/
if (unlikely(csg_id < 0))
return 0;
if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) {
ret = -EINVAL;
} else {
/* We do the allocation without holding the scheduler lock to avoid
* blocking the scheduling.
*/
ret = panthor_heap_grow(heaps, heap_address,
renderpasses_in_flight,
pending_frag_count, &new_chunk_va);
}
/* If the heap context doesn't have memory for us, we want to let the
* FW try to reclaim memory by waiting for fragment jobs to land or by
* executing the tiler OOM exception handler, which is supposed to
* implement incremental rendering.
*/
if (ret && ret != -ENOMEM) {
drm_warn(&ptdev->base, "Failed to extend the tiler heap\n");
group->fatal_queues |= BIT(cs_id);
sched_queue_delayed_work(sched, tick, 0);
goto out_put_heap_pool;
}
mutex_lock(&sched->lock);
csg_id = group->csg_id;
if (csg_id >= 0) {
struct panthor_fw_csg_iface *csg_iface;
struct panthor_fw_cs_iface *cs_iface;
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
cs_iface->input->heap_start = new_chunk_va;
cs_iface->input->heap_end = new_chunk_va;
panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM);
panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id));
panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
}
mutex_unlock(&sched->lock);
/* We allocated a chunck, but couldn't link it to the heap
* context because the group was scheduled out while we were
* allocating memory. We need to return this chunk to the heap.
*/
if (unlikely(csg_id < 0 && new_chunk_va))
panthor_heap_return_chunk(heaps, heap_address, new_chunk_va);
ret = 0;
out_put_heap_pool:
panthor_heap_pool_put(heaps);
return ret;
}
static void group_tiler_oom_work(struct work_struct *work)
{
struct panthor_group *group =
container_of(work, struct panthor_group, tiler_oom_work);
u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0);
while (tiler_oom) {
u32 cs_id = ffs(tiler_oom) - 1;
group_process_tiler_oom(group, cs_id);
tiler_oom &= ~BIT(cs_id);
}
group_put(group);
}
static void
cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev,
u32 csg_id, u32 cs_id)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
struct panthor_group *group = csg_slot->group;
lockdep_assert_held(&sched->lock);
if (drm_WARN_ON(&ptdev->base, !group))
return;
atomic_or(BIT(cs_id), &group->tiler_oom);
/* We don't use group_queue_work() here because we want to queue the
* work item to the heap_alloc_wq.
*/
group_get(group);
if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work))
group_put(group);
}
static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,
u32 csg_id, u32 cs_id)
{
struct panthor_fw_cs_iface *cs_iface;
u32 req, ack, events;
lockdep_assert_held(&ptdev->scheduler->lock);
cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
req = cs_iface->input->req;
ack = cs_iface->output->ack;
events = (req ^ ack) & CS_EVT_MASK;
if (events & CS_FATAL)
cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id);
if (events & CS_FAULT)
cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id);
if (events & CS_TILER_OOM)
cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id);
/* We don't acknowledge the TILER_OOM event since its handling is
* deferred to a separate work.
*/
panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT);
return (events & (CS_FAULT | CS_TILER_OOM)) != 0;
}
static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id)
{
struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
struct panthor_fw_csg_iface *csg_iface;
lockdep_assert_held(&ptdev->scheduler->lock);
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE;
}
static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id)
{
struct panthor_scheduler *sched = ptdev->scheduler;
lockdep_assert_held(&sched->lock);
sched->might_have_idle_groups = true;
/* Schedule a tick so we can evict idle groups and schedule non-idle
* ones. This will also update runtime PM and devfreq busy/idle states,
* so the device can lower its frequency or get suspended.
*/
sched_queue_delayed_work(sched, tick, 0);
}
static void csg_slot_sync_update_locked(struct panthor_device *ptdev,
u32 csg_id)
{
struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
struct panthor_group *group = csg_slot->group;
lockdep_assert_held(&ptdev->scheduler->lock);
if (group)
group_queue_work(group, sync_upd);
sched_queue_work(ptdev->scheduler, sync_upd);
}
static void
csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
struct panthor_group *group = csg_slot->group;
lockdep_assert_held(&sched->lock);
drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id);
group = csg_slot->group;
if (!drm_WARN_ON(&ptdev->base, !group))
group->timedout = true;
sched_queue_delayed_work(sched, tick, 0);
}
static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id)
{
u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events;
struct panthor_fw_csg_iface *csg_iface;
u32 ring_cs_db_mask = 0;
lockdep_assert_held(&ptdev->scheduler->lock);
if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
return;
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
req = READ_ONCE(csg_iface->input->req);
ack = READ_ONCE(csg_iface->output->ack);
cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req);
cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack);
csg_events = (req ^ ack) & CSG_EVT_MASK;
/* There may not be any pending CSG/CS interrupts to process */
if (req == ack && cs_irq_req == cs_irq_ack)
return;
/* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before
* examining the CS_ACK & CS_REQ bits. This would ensure that Host
* doesn't miss an interrupt for the CS in the race scenario where
* whilst Host is servicing an interrupt for the CS, firmware sends
* another interrupt for that CS.
*/
csg_iface->input->cs_irq_ack = cs_irq_req;
panthor_fw_update_reqs(csg_iface, req, ack,
CSG_SYNC_UPDATE |
CSG_IDLE |
CSG_PROGRESS_TIMER_EVENT);
if (csg_events & CSG_IDLE)
csg_slot_process_idle_event_locked(ptdev, csg_id);
if (csg_events & CSG_PROGRESS_TIMER_EVENT)
csg_slot_process_progress_timer_event_locked(ptdev, csg_id);
cs_irqs = cs_irq_req ^ cs_irq_ack;
while (cs_irqs) {
u32 cs_id = ffs(cs_irqs) - 1;
if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id))
ring_cs_db_mask |= BIT(cs_id);
cs_irqs &= ~BIT(cs_id);
}
if (csg_events & CSG_SYNC_UPDATE)
csg_slot_sync_update_locked(ptdev, csg_id);
if (ring_cs_db_mask)
panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask);
panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
}
static void sched_process_idle_event_locked(struct panthor_device *ptdev)
{
struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
lockdep_assert_held(&ptdev->scheduler->lock);
/* Acknowledge the idle event and schedule a tick. */
panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE);
sched_queue_delayed_work(ptdev->scheduler, tick, 0);
}
/**
* sched_process_global_irq_locked() - Process the scheduling part of a global IRQ
* @ptdev: Device.
*/
static void sched_process_global_irq_locked(struct panthor_device *ptdev)
{
struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
u32 req, ack, evts;
lockdep_assert_held(&ptdev->scheduler->lock);
req = READ_ONCE(glb_iface->input->req);
ack = READ_ONCE(glb_iface->output->ack);
evts = (req ^ ack) & GLB_EVT_MASK;
if (evts & GLB_IDLE)
sched_process_idle_event_locked(ptdev);
}
static void process_fw_events_work(struct work_struct *work)
{
struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
fw_events_work);
u32 events = atomic_xchg(&sched->fw_events, 0);
struct panthor_device *ptdev = sched->ptdev;
mutex_lock(&sched->lock);
if (events & JOB_INT_GLOBAL_IF) {
sched_process_global_irq_locked(ptdev);
events &= ~JOB_INT_GLOBAL_IF;
}
while (events) {
u32 csg_id = ffs(events) - 1;
sched_process_csg_irq_locked(ptdev, csg_id);
events &= ~BIT(csg_id);
}
mutex_unlock(&sched->lock);
}
/**
* panthor_sched_report_fw_events() - Report FW events to the scheduler.
*/
void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
{
if (!ptdev->scheduler)
return;
atomic_or(events, &ptdev->scheduler->fw_events);
sched_queue_work(ptdev->scheduler, fw_events);
}
static const char *fence_get_driver_name(struct dma_fence *fence)
{
return "panthor";
}
static const char *queue_fence_get_timeline_name(struct dma_fence *fence)
{
return "queue-fence";
}
static const struct dma_fence_ops panthor_queue_fence_ops = {
.get_driver_name = fence_get_driver_name,
.get_timeline_name = queue_fence_get_timeline_name,
};
struct panthor_csg_slots_upd_ctx {
u32 update_mask;
u32 timedout_mask;
struct {
u32 value;
u32 mask;
} requests[MAX_CSGS];
};
static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx)
{
memset(ctx, 0, sizeof(*ctx));
}
static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev,
struct panthor_csg_slots_upd_ctx *ctx,
u32 csg_id, u32 value, u32 mask)
{
if (drm_WARN_ON(&ptdev->base, !mask) ||
drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
return;
ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask);
ctx->requests[csg_id].mask |= mask;
ctx->update_mask |= BIT(csg_id);
}
static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev,
struct panthor_csg_slots_upd_ctx *ctx)
{
struct panthor_scheduler *sched = ptdev->scheduler;
u32 update_slots = ctx->update_mask;
lockdep_assert_held(&sched->lock);
if (!ctx->update_mask)
return 0;
while (update_slots) {
struct panthor_fw_csg_iface *csg_iface;
u32 csg_id = ffs(update_slots) - 1;
update_slots &= ~BIT(csg_id);
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
panthor_fw_update_reqs(csg_iface, req,
ctx->requests[csg_id].value,
ctx->requests[csg_id].mask);
}
panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask);
update_slots = ctx->update_mask;
while (update_slots) {
struct panthor_fw_csg_iface *csg_iface;
u32 csg_id = ffs(update_slots) - 1;
u32 req_mask = ctx->requests[csg_id].mask, acked;
int ret;
update_slots &= ~BIT(csg_id);
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100);
if (acked & CSG_ENDPOINT_CONFIG)
csg_slot_sync_priority_locked(ptdev, csg_id);
if (acked & CSG_STATE_MASK)
csg_slot_sync_state_locked(ptdev, csg_id);
if (acked & CSG_STATUS_UPDATE) {
csg_slot_sync_queues_state_locked(ptdev, csg_id);
csg_slot_sync_idle_state_locked(ptdev, csg_id);
}
if (ret && acked != req_mask &&
((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) {
drm_err(&ptdev->base, "CSG %d update request timedout", csg_id);
ctx->timedout_mask |= BIT(csg_id);
}
}
if (ctx->timedout_mask)
return -ETIMEDOUT;
return 0;
}
struct panthor_sched_tick_ctx {
struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT];
struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT];
u32 idle_group_count;
u32 group_count;
enum panthor_csg_priority min_priority;
struct panthor_vm *vms[MAX_CS_PER_CSG];
u32 as_count;
bool immediate_tick;
u32 csg_upd_failed_mask;
};
static bool
tick_ctx_is_full(const struct panthor_scheduler *sched,
const struct panthor_sched_tick_ctx *ctx)
{
return ctx->group_count == sched->csg_slot_count;
}
static bool
group_is_idle(struct panthor_group *group)
{
struct panthor_device *ptdev = group->ptdev;
u32 inactive_queues;
if (group->csg_id >= 0)
return ptdev->scheduler->csg_slots[group->csg_id].idle;
inactive_queues = group->idle_queues | group->blocked_queues;
return hweight32(inactive_queues) == group->queue_count;
}
static bool
group_can_run(struct panthor_group *group)
{
return group->state != PANTHOR_CS_GROUP_TERMINATED &&
group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE &&
!group->destroyed && group->fatal_queues == 0 &&
!group->timedout;
}
static void
tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,
struct panthor_sched_tick_ctx *ctx,
struct list_head *queue,
bool skip_idle_groups,
bool owned_by_tick_ctx)
{
struct panthor_group *group, *tmp;
if (tick_ctx_is_full(sched, ctx))
return;
list_for_each_entry_safe(group, tmp, queue, run_node) {
u32 i;
if (!group_can_run(group))
continue;
if (skip_idle_groups && group_is_idle(group))
continue;
for (i = 0; i < ctx->as_count; i++) {
if (ctx->vms[i] == group->vm)
break;
}
if (i == ctx->as_count && ctx->as_count == sched->as_slot_count)
continue;
if (!owned_by_tick_ctx)
group_get(group);
list_move_tail(&group->run_node, &ctx->groups[group->priority]);
ctx->group_count++;
if (group_is_idle(group))
ctx->idle_group_count++;
if (i == ctx->as_count)
ctx->vms[ctx->as_count++] = group->vm;
if (ctx->min_priority > group->priority)
ctx->min_priority = group->priority;
if (tick_ctx_is_full(sched, ctx))
return;
}
}
static void
tick_ctx_insert_old_group(struct panthor_scheduler *sched,
struct panthor_sched_tick_ctx *ctx,
struct panthor_group *group,
bool full_tick)
{
struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id];
struct panthor_group *other_group;
if (!full_tick) {
list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
return;
}
/* Rotate to make sure groups with lower CSG slot
* priorities have a chance to get a higher CSG slot
* priority next time they get picked. This priority
* has an impact on resource request ordering, so it's
* important to make sure we don't let one group starve
* all other groups with the same group priority.
*/
list_for_each_entry(other_group,
&ctx->old_groups[csg_slot->group->priority],
run_node) {
struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id];
if (other_csg_slot->priority > csg_slot->priority) {
list_add_tail(&csg_slot->group->run_node, &other_group->run_node);
return;
}
}
list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
}
static void
tick_ctx_init(struct panthor_scheduler *sched,
struct panthor_sched_tick_ctx *ctx,
bool full_tick)
{
struct panthor_device *ptdev = sched->ptdev;
struct panthor_csg_slots_upd_ctx upd_ctx;
int ret;
u32 i;
memset(ctx, 0, sizeof(*ctx));
csgs_upd_ctx_init(&upd_ctx);
ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT;
for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
INIT_LIST_HEAD(&ctx->groups[i]);
INIT_LIST_HEAD(&ctx->old_groups[i]);
}
for (i = 0; i < sched->csg_slot_count; i++) {
struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
struct panthor_group *group = csg_slot->group;
struct panthor_fw_csg_iface *csg_iface;
if (!group)
continue;
csg_iface = panthor_fw_get_csg_iface(ptdev, i);
group_get(group);
/* If there was unhandled faults on the VM, force processing of
* CSG IRQs, so we can flag the faulty queue.
*/
if (panthor_vm_has_unhandled_faults(group->vm)) {
sched_process_csg_irq_locked(ptdev, i);
/* No fatal fault reported, flag all queues as faulty. */
if (!group->fatal_queues)
group->fatal_queues |= GENMASK(group->queue_count - 1, 0);
}
tick_ctx_insert_old_group(sched, ctx, group, full_tick);
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
csg_iface->output->ack ^ CSG_STATUS_UPDATE,
CSG_STATUS_UPDATE);
}
ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
if (ret) {
panthor_device_schedule_reset(ptdev);
ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
}
}
#define NUM_INSTRS_PER_SLOT 16
static void
group_term_post_processing(struct panthor_group *group)
{
struct panthor_job *job, *tmp;
LIST_HEAD(faulty_jobs);
bool cookie;
u32 i = 0;
if (drm_WARN_ON(&group->ptdev->base, group_can_run(group)))
return;
cookie = dma_fence_begin_signalling();
for (i = 0; i < group->queue_count; i++) {
struct panthor_queue *queue = group->queues[i];
struct panthor_syncobj_64b *syncobj;
int err;
if (group->fatal_queues & BIT(i))
err = -EINVAL;
else if (group->timedout)
err = -ETIMEDOUT;
else
err = -ECANCELED;
if (!queue)
continue;
spin_lock(&queue->fence_ctx.lock);
list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) {
list_move_tail(&job->node, &faulty_jobs);
dma_fence_set_error(job->done_fence, err);
dma_fence_signal_locked(job->done_fence);
}
spin_unlock(&queue->fence_ctx.lock);
/* Manually update the syncobj seqno to unblock waiters. */
syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj));
syncobj->status = ~0;
syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno);
sched_queue_work(group->ptdev->scheduler, sync_upd);
}
dma_fence_end_signalling(cookie);
list_for_each_entry_safe(job, tmp, &faulty_jobs, node) {
list_del_init(&job->node);
panthor_job_put(&job->base);
}
}
static void group_term_work(struct work_struct *work)
{
struct panthor_group *group =
container_of(work, struct panthor_group, term_work);
group_term_post_processing(group);
group_put(group);
}
static void
tick_ctx_cleanup(struct panthor_scheduler *sched,
struct panthor_sched_tick_ctx *ctx)
{
struct panthor_device *ptdev = sched->ptdev;
struct panthor_group *group, *tmp;
u32 i;
for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) {
list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) {
/* If everything went fine, we should only have groups
* to be terminated in the old_groups lists.
*/
drm_WARN_ON(&ptdev->base, !ctx->csg_upd_failed_mask &&
group_can_run(group));
if (!group_can_run(group)) {
list_del_init(&group->run_node);
list_del_init(&group->wait_node);
group_queue_work(group, term);
} else if (group->csg_id >= 0) {
list_del_init(&group->run_node);
} else {
list_move(&group->run_node,
group_is_idle(group) ?
&sched->groups.idle[group->priority] :
&sched->groups.runnable[group->priority]);
}
group_put(group);
}
}
for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
/* If everything went fine, the groups to schedule lists should
* be empty.
*/
drm_WARN_ON(&ptdev->base,
!ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i]));
list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) {
if (group->csg_id >= 0) {
list_del_init(&group->run_node);
} else {
list_move(&group->run_node,
group_is_idle(group) ?
&sched->groups.idle[group->priority] :
&sched->groups.runnable[group->priority]);
}
group_put(group);
}
}
}
static void
tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx)
{
struct panthor_group *group, *tmp;
struct panthor_device *ptdev = sched->ptdev;
struct panthor_csg_slot *csg_slot;
int prio, new_csg_prio = MAX_CSG_PRIO, i;
u32 free_csg_slots = 0;
struct panthor_csg_slots_upd_ctx upd_ctx;
int ret;
csgs_upd_ctx_init(&upd_ctx);
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
/* Suspend or terminate evicted groups. */
list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
bool term = !group_can_run(group);
int csg_id = group->csg_id;
if (drm_WARN_ON(&ptdev->base, csg_id < 0))
continue;
csg_slot = &sched->csg_slots[csg_id];
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND,
CSG_STATE_MASK);
}
/* Update priorities on already running groups. */
list_for_each_entry(group, &ctx->groups[prio], run_node) {
struct panthor_fw_csg_iface *csg_iface;
int csg_id = group->csg_id;
if (csg_id < 0) {
new_csg_prio--;
continue;
}
csg_slot = &sched->csg_slots[csg_id];
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
if (csg_slot->priority == new_csg_prio) {
new_csg_prio--;
continue;
}
panthor_fw_update_reqs(csg_iface, endpoint_req,
CSG_EP_REQ_PRIORITY(new_csg_prio),
CSG_EP_REQ_PRIORITY_MASK);
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
CSG_ENDPOINT_CONFIG);
new_csg_prio--;
}
}
ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
if (ret) {
panthor_device_schedule_reset(ptdev);
ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
return;
}
/* Unbind evicted groups. */
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
/* This group is gone. Process interrupts to clear
* any pending interrupts before we start the new
* group.
*/
if (group->csg_id >= 0)
sched_process_csg_irq_locked(ptdev, group->csg_id);
group_unbind_locked(group);
}
}
for (i = 0; i < sched->csg_slot_count; i++) {
if (!sched->csg_slots[i].group)
free_csg_slots |= BIT(i);
}
csgs_upd_ctx_init(&upd_ctx);
new_csg_prio = MAX_CSG_PRIO;
/* Start new groups. */
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
list_for_each_entry(group, &ctx->groups[prio], run_node) {
int csg_id = group->csg_id;
struct panthor_fw_csg_iface *csg_iface;
if (csg_id >= 0) {
new_csg_prio--;
continue;
}
csg_id = ffs(free_csg_slots) - 1;
if (drm_WARN_ON(&ptdev->base, csg_id < 0))
break;
csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
csg_slot = &sched->csg_slots[csg_id];
group_bind_locked(group, csg_id);
csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--);
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
group->state == PANTHOR_CS_GROUP_SUSPENDED ?
CSG_STATE_RESUME : CSG_STATE_START,
CSG_STATE_MASK);
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
CSG_ENDPOINT_CONFIG);
free_csg_slots &= ~BIT(csg_id);
}
}
ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
if (ret) {
panthor_device_schedule_reset(ptdev);
ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
return;
}
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) {
list_del_init(&group->run_node);
/* If the group has been destroyed while we were
* scheduling, ask for an immediate tick to
* re-evaluate as soon as possible and get rid of
* this dangling group.
*/
if (group->destroyed)
ctx->immediate_tick = true;
group_put(group);
}
/* Return evicted groups to the idle or run queues. Groups
* that can no longer be run (because they've been destroyed
* or experienced an unrecoverable error) will be scheduled
* for destruction in tick_ctx_cleanup().
*/
list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) {
if (!group_can_run(group))
continue;
if (group_is_idle(group))
list_move_tail(&group->run_node, &sched->groups.idle[prio]);
else
list_move_tail(&group->run_node, &sched->groups.runnable[prio]);
group_put(group);
}
}
sched->used_csg_slot_count = ctx->group_count;
sched->might_have_idle_groups = ctx->idle_group_count > 0;
}
static u64
tick_ctx_update_resched_target(struct panthor_scheduler *sched,
const struct panthor_sched_tick_ctx *ctx)
{
/* We had space left, no need to reschedule until some external event happens. */
if (!tick_ctx_is_full(sched, ctx))
goto no_tick;
/* If idle groups were scheduled, no need to wake up until some external
* event happens (group unblocked, new job submitted, ...).
*/
if (ctx->idle_group_count)
goto no_tick;
if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT))
goto no_tick;
/* If there are groups of the same priority waiting, we need to
* keep the scheduler ticking, otherwise, we'll just wait for
* new groups with higher priority to be queued.
*/
if (!list_empty(&sched->groups.runnable[ctx->min_priority])) {
u64 resched_target = sched->last_tick + sched->tick_period;
if (time_before64(sched->resched_target, sched->last_tick) ||
time_before64(resched_target, sched->resched_target))
sched->resched_target = resched_target;
return sched->resched_target - sched->last_tick;
}
no_tick:
sched->resched_target = U64_MAX;
return U64_MAX;
}
static void tick_work(struct work_struct *work)
{
struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
tick_work.work);
struct panthor_device *ptdev = sched->ptdev;
struct panthor_sched_tick_ctx ctx;
u64 remaining_jiffies = 0, resched_delay;
u64 now = get_jiffies_64();
int prio, ret, cookie;
if (!drm_dev_enter(&ptdev->base, &cookie))
return;
ret = pm_runtime_resume_and_get(ptdev->base.dev);
if (drm_WARN_ON(&ptdev->base, ret))
goto out_dev_exit;
if (time_before64(now, sched->resched_target))
remaining_jiffies = sched->resched_target - now;
mutex_lock(&sched->lock);
if (panthor_device_reset_is_pending(sched->ptdev))
goto out_unlock;
tick_ctx_init(sched, &ctx, remaining_jiffies != 0);
if (ctx.csg_upd_failed_mask)
goto out_cleanup_ctx;
if (remaining_jiffies) {
/* Scheduling forced in the middle of a tick. Only RT groups
* can preempt non-RT ones. Currently running RT groups can't be
* preempted.
*/
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
prio >= 0 && !tick_ctx_is_full(sched, &ctx);
prio--) {
tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio],
true, true);
if (prio == PANTHOR_CSG_PRIORITY_RT) {
tick_ctx_pick_groups_from_list(sched, &ctx,
&sched->groups.runnable[prio],
true, false);
}
}
}
/* First pick non-idle groups */
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
prio >= 0 && !tick_ctx_is_full(sched, &ctx);
prio--) {
tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio],
true, false);
tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);
}
/* If we have free CSG slots left, pick idle groups */
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
prio >= 0 && !tick_ctx_is_full(sched, &ctx);
prio--) {
/* Check the old_group queue first to avoid reprogramming the slots */
tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true);
tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio],
false, false);
}
tick_ctx_apply(sched, &ctx);
if (ctx.csg_upd_failed_mask)
goto out_cleanup_ctx;
if (ctx.idle_group_count == ctx.group_count) {
panthor_devfreq_record_idle(sched->ptdev);
if (sched->pm.has_ref) {
pm_runtime_put_autosuspend(ptdev->base.dev);
sched->pm.has_ref = false;
}
} else {
panthor_devfreq_record_busy(sched->ptdev);
if (!sched->pm.has_ref) {
pm_runtime_get(ptdev->base.dev);
sched->pm.has_ref = true;
}
}
sched->last_tick = now;
resched_delay = tick_ctx_update_resched_target(sched, &ctx);
if (ctx.immediate_tick)
resched_delay = 0;
if (resched_delay != U64_MAX)
sched_queue_delayed_work(sched, tick, resched_delay);
out_cleanup_ctx:
tick_ctx_cleanup(sched, &ctx);
out_unlock:
mutex_unlock(&sched->lock);
pm_runtime_mark_last_busy(ptdev->base.dev);
pm_runtime_put_autosuspend(ptdev->base.dev);
out_dev_exit:
drm_dev_exit(cookie);
}
static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx)
{
struct panthor_queue *queue = group->queues[queue_idx];
union {
struct panthor_syncobj_64b sync64;
struct panthor_syncobj_32b sync32;
} *syncobj;
bool result;
u64 value;
syncobj = panthor_queue_get_syncwait_obj(group, queue);
if (!syncobj)
return -EINVAL;
value = queue->syncwait.sync64 ?
syncobj->sync64.seqno :
syncobj->sync32.seqno;
if (queue->syncwait.gt)
result = value > queue->syncwait.ref;
else
result = value <= queue->syncwait.ref;
if (result)
panthor_queue_put_syncwait_obj(queue);
return result;
}
static void sync_upd_work(struct work_struct *work)
{
struct panthor_scheduler *sched = container_of(work,
struct panthor_scheduler,
sync_upd_work);
struct panthor_group *group, *tmp;
bool immediate_tick = false;
mutex_lock(&sched->lock);
list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) {
u32 tested_queues = group->blocked_queues;
u32 unblocked_queues = 0;
while (tested_queues) {
u32 cs_id = ffs(tested_queues) - 1;
int ret;
ret = panthor_queue_eval_syncwait(group, cs_id);
drm_WARN_ON(&group->ptdev->base, ret < 0);
if (ret)
unblocked_queues |= BIT(cs_id);
tested_queues &= ~BIT(cs_id);
}
if (unblocked_queues) {
group->blocked_queues &= ~unblocked_queues;
if (group->csg_id < 0) {
list_move(&group->run_node,
&sched->groups.runnable[group->priority]);
if (group->priority == PANTHOR_CSG_PRIORITY_RT)
immediate_tick = true;
}
}
if (!group->blocked_queues)
list_del_init(&group->wait_node);
}
mutex_unlock(&sched->lock);
if (immediate_tick)
sched_queue_delayed_work(sched, tick, 0);
}
static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)
{
struct panthor_device *ptdev = group->ptdev;
struct panthor_scheduler *sched = ptdev->scheduler;
struct list_head *queue = &sched->groups.runnable[group->priority];
u64 delay_jiffies = 0;
bool was_idle;
u64 now;
if (!group_can_run(group))
return;
/* All updated queues are blocked, no need to wake up the scheduler. */
if ((queue_mask & group->blocked_queues) == queue_mask)
return;
was_idle = group_is_idle(group);
group->idle_queues &= ~queue_mask;
/* Don't mess up with the lists if we're in a middle of a reset. */
if (atomic_read(&sched->reset.in_progress))
return;
if (was_idle && !group_is_idle(group))
list_move_tail(&group->run_node, queue);
/* RT groups are preemptive. */
if (group->priority == PANTHOR_CSG_PRIORITY_RT) {
sched_queue_delayed_work(sched, tick, 0);
return;
}
/* Some groups might be idle, force an immediate tick to
* re-evaluate.
*/
if (sched->might_have_idle_groups) {
sched_queue_delayed_work(sched, tick, 0);
return;
}
/* Scheduler is ticking, nothing to do. */
if (sched->resched_target != U64_MAX) {
/* If there are free slots, force immediating ticking. */
if (sched->used_csg_slot_count < sched->csg_slot_count)
sched_queue_delayed_work(sched, tick, 0);
return;
}
/* Scheduler tick was off, recalculate the resched_target based on the
* last tick event, and queue the scheduler work.
*/
now = get_jiffies_64();
sched->resched_target = sched->last_tick + sched->tick_period;
if (sched->used_csg_slot_count == sched->csg_slot_count &&
time_before64(now, sched->resched_target))
delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);
sched_queue_delayed_work(sched, tick, delay_jiffies);
}
static void queue_stop(struct panthor_queue *queue,
struct panthor_job *bad_job)
{
drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);
}
static void queue_start(struct panthor_queue *queue)
{
struct panthor_job *job;
/* Re-assign the parent fences. */
list_for_each_entry(job, &queue->scheduler.pending_list, base.list)
job->base.s_fence->parent = dma_fence_get(job->done_fence);
drm_sched_start(&queue->scheduler);
}
static void panthor_group_stop(struct panthor_group *group)
{
struct panthor_scheduler *sched = group->ptdev->scheduler;
lockdep_assert_held(&sched->reset.lock);
for (u32 i = 0; i < group->queue_count; i++)
queue_stop(group->queues[i], NULL);
group_get(group);
list_move_tail(&group->run_node, &sched->reset.stopped_groups);
}
static void panthor_group_start(struct panthor_group *group)
{
struct panthor_scheduler *sched = group->ptdev->scheduler;
lockdep_assert_held(&group->ptdev->scheduler->reset.lock);
for (u32 i = 0; i < group->queue_count; i++)
queue_start(group->queues[i]);
if (group_can_run(group)) {
list_move_tail(&group->run_node,
group_is_idle(group) ?
&sched->groups.idle[group->priority] :
&sched->groups.runnable[group->priority]);
} else {
list_del_init(&group->run_node);
list_del_init(&group->wait_node);
group_queue_work(group, term);
}
group_put(group);
}
static void panthor_sched_immediate_tick(struct panthor_device *ptdev)
{
struct panthor_scheduler *sched = ptdev->scheduler;
sched_queue_delayed_work(sched, tick, 0);
}
/**
* panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.
*/
void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)
{
/* Force a tick to immediately kill faulty groups. */
if (ptdev->scheduler)
panthor_sched_immediate_tick(ptdev);
}
void panthor_sched_resume(struct panthor_device *ptdev)
{
/* Force a tick to re-evaluate after a resume. */
panthor_sched_immediate_tick(ptdev);
}
void panthor_sched_suspend(struct panthor_device *ptdev)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_csg_slots_upd_ctx upd_ctx;
struct panthor_group *group;
u32 suspended_slots;
u32 i;
mutex_lock(&sched->lock);
csgs_upd_ctx_init(&upd_ctx);
for (i = 0; i < sched->csg_slot_count; i++) {
struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
if (csg_slot->group) {
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
group_can_run(csg_slot->group) ?
CSG_STATE_SUSPEND : CSG_STATE_TERMINATE,
CSG_STATE_MASK);
}
}
suspended_slots = upd_ctx.update_mask;
csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
suspended_slots &= ~upd_ctx.timedout_mask;
if (upd_ctx.timedout_mask) {
u32 slot_mask = upd_ctx.timedout_mask;
drm_err(&ptdev->base, "CSG suspend failed, escalating to termination");
csgs_upd_ctx_init(&upd_ctx);
while (slot_mask) {
u32 csg_id = ffs(slot_mask) - 1;
csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
CSG_STATE_TERMINATE,
CSG_STATE_MASK);
slot_mask &= ~BIT(csg_id);
}
csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
slot_mask = upd_ctx.timedout_mask;
while (slot_mask) {
u32 csg_id = ffs(slot_mask) - 1;
struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
/* Terminate command timedout, but the soft-reset will
* automatically terminate all active groups, so let's
* force the state to halted here.
*/
if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED)
csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
slot_mask &= ~BIT(csg_id);
}
}
/* Flush L2 and LSC caches to make sure suspend state is up-to-date.
* If the flush fails, flag all queues for termination.
*/
if (suspended_slots) {
bool flush_caches_failed = false;
u32 slot_mask = suspended_slots;
if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0))
flush_caches_failed = true;
while (slot_mask) {
u32 csg_id = ffs(slot_mask) - 1;
struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
if (flush_caches_failed)
csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
else
csg_slot_sync_update_locked(ptdev, csg_id);
slot_mask &= ~BIT(csg_id);
}
}
for (i = 0; i < sched->csg_slot_count; i++) {
struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
group = csg_slot->group;
if (!group)
continue;
group_get(group);
if (group->csg_id >= 0)
sched_process_csg_irq_locked(ptdev, group->csg_id);
group_unbind_locked(group);
drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node));
if (group_can_run(group)) {
list_add(&group->run_node,
&sched->groups.idle[group->priority]);
} else {
/* We don't bother stopping the scheduler if the group is
* faulty, the group termination work will finish the job.
*/
list_del_init(&group->wait_node);
group_queue_work(group, term);
}
group_put(group);
}
mutex_unlock(&sched->lock);
}
void panthor_sched_pre_reset(struct panthor_device *ptdev)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_group *group, *group_tmp;
u32 i;
mutex_lock(&sched->reset.lock);
atomic_set(&sched->reset.in_progress, true);
/* Cancel all scheduler works. Once this is done, these works can't be
* scheduled again until the reset operation is complete.
*/
cancel_work_sync(&sched->sync_upd_work);
cancel_delayed_work_sync(&sched->tick_work);
panthor_sched_suspend(ptdev);
/* Stop all groups that might still accept jobs, so we don't get passed
* new jobs while we're resetting.
*/
for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) {
/* All groups should be in the idle lists. */
drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i]));
list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node)
panthor_group_stop(group);
}
for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) {
list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node)
panthor_group_stop(group);
}
mutex_unlock(&sched->reset.lock);
}
void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed)
{
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_group *group, *group_tmp;
mutex_lock(&sched->reset.lock);
list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) {
/* Consider all previously running group as terminated if the
* reset failed.
*/
if (reset_failed)
group->state = PANTHOR_CS_GROUP_TERMINATED;
panthor_group_start(group);
}
/* We're done resetting the GPU, clear the reset.in_progress bit so we can
* kick the scheduler.
*/
atomic_set(&sched->reset.in_progress, false);
mutex_unlock(&sched->reset.lock);
/* No need to queue a tick and update syncs if the reset failed. */
if (!reset_failed) {
sched_queue_delayed_work(sched, tick, 0);
sched_queue_work(sched, sync_upd);
}
}
static void group_sync_upd_work(struct work_struct *work)
{
struct panthor_group *group =
container_of(work, struct panthor_group, sync_upd_work);
struct panthor_job *job, *job_tmp;
LIST_HEAD(done_jobs);
u32 queue_idx;
bool cookie;
cookie = dma_fence_begin_signalling();
for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
struct panthor_queue *queue = group->queues[queue_idx];
struct panthor_syncobj_64b *syncobj;
if (!queue)
continue;
syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj));
spin_lock(&queue->fence_ctx.lock);
list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
if (syncobj->seqno < job->done_fence->seqno)
break;
list_move_tail(&job->node, &done_jobs);
dma_fence_signal_locked(job->done_fence);
}
spin_unlock(&queue->fence_ctx.lock);
}
dma_fence_end_signalling(cookie);
list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {
list_del_init(&job->node);
panthor_job_put(&job->base);
}
group_put(group);
}
static struct dma_fence *
queue_run_job(struct drm_sched_job *sched_job)
{
struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
struct panthor_group *group = job->group;
struct panthor_queue *queue = group->queues[job->queue_idx];
struct panthor_device *ptdev = group->ptdev;
struct panthor_scheduler *sched = ptdev->scheduler;
u32 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
u32 ringbuf_insert = queue->iface.input->insert & (ringbuf_size - 1);
u64 addr_reg = ptdev->csif_info.cs_reg_count -
ptdev->csif_info.unpreserved_cs_reg_count;
u64 val_reg = addr_reg + 2;
u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) +
job->queue_idx * sizeof(struct panthor_syncobj_64b);
u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0);
struct dma_fence *done_fence;
int ret;
u64 call_instrs[NUM_INSTRS_PER_SLOT] = {
/* MOV32 rX+2, cs.latest_flush */
(2ull << 56) | (val_reg << 48) | job->call_info.latest_flush,
/* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */
(36ull << 56) | (0ull << 48) | (val_reg << 40) | (0 << 16) | 0x233,
/* MOV48 rX:rX+1, cs.start */
(1ull << 56) | (addr_reg << 48) | job->call_info.start,
/* MOV32 rX+2, cs.size */
(2ull << 56) | (val_reg << 48) | job->call_info.size,
/* WAIT(0) => waits for FLUSH_CACHE2 instruction */
(3ull << 56) | (1 << 16),
/* CALL rX:rX+1, rX+2 */
(32ull << 56) | (addr_reg << 40) | (val_reg << 32),
/* MOV48 rX:rX+1, sync_addr */
(1ull << 56) | (addr_reg << 48) | sync_addr,
/* MOV48 rX+2, #1 */
(1ull << 56) | (val_reg << 48) | 1,
/* WAIT(all) */
(3ull << 56) | (waitall_mask << 16),
/* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/
(51ull << 56) | (0ull << 48) | (addr_reg << 40) | (val_reg << 32) | (0 << 16) | 1,
/* ERROR_BARRIER, so we can recover from faults at job
* boundaries.
*/
(47ull << 56),
};
/* Need to be cacheline aligned to please the prefetcher. */
static_assert(sizeof(call_instrs) % 64 == 0,
"call_instrs is not aligned on a cacheline");
/* Stream size is zero, nothing to do except making sure all previously
* submitted jobs are done before we signal the
* drm_sched_job::s_fence::finished fence.
*/
if (!job->call_info.size) {
job->done_fence = dma_fence_get(queue->fence_ctx.last_fence);
return dma_fence_get(job->done_fence);
}
ret = pm_runtime_resume_and_get(ptdev->base.dev);
if (drm_WARN_ON(&ptdev->base, ret))
return ERR_PTR(ret);
mutex_lock(&sched->lock);
if (!group_can_run(group)) {
done_fence = ERR_PTR(-ECANCELED);
goto out_unlock;
}
dma_fence_init(job->done_fence,
&panthor_queue_fence_ops,
&queue->fence_ctx.lock,
queue->fence_ctx.id,
atomic64_inc_return(&queue->fence_ctx.seqno));
memcpy(queue->ringbuf->kmap + ringbuf_insert,
call_instrs, sizeof(call_instrs));
panthor_job_get(&job->base);
spin_lock(&queue->fence_ctx.lock);
list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs);
spin_unlock(&queue->fence_ctx.lock);
job->ringbuf.start = queue->iface.input->insert;
job->ringbuf.end = job->ringbuf.start + sizeof(call_instrs);
/* Make sure the ring buffer is updated before the INSERT
* register.
*/
wmb();
queue->iface.input->extract = queue->iface.output->extract;
queue->iface.input->insert = job->ringbuf.end;
if (group->csg_id < 0) {
/* If the queue is blocked, we want to keep the timeout running, so we
* can detect unbounded waits and kill the group when that happens.
* Otherwise, we suspend the timeout so the time we spend waiting for
* a CSG slot is not counted.
*/
if (!(group->blocked_queues & BIT(job->queue_idx)) &&
!queue->timeout_suspended) {
queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
queue->timeout_suspended = true;
}
group_schedule_locked(group, BIT(job->queue_idx));
} else {
gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);
if (!sched->pm.has_ref &&
!(group->blocked_queues & BIT(job->queue_idx))) {
pm_runtime_get(ptdev->base.dev);
sched->pm.has_ref = true;
}
panthor_devfreq_record_busy(sched->ptdev);
}
/* Update the last fence. */
dma_fence_put(queue->fence_ctx.last_fence);
queue->fence_ctx.last_fence = dma_fence_get(job->done_fence);
done_fence = dma_fence_get(job->done_fence);
out_unlock:
mutex_unlock(&sched->lock);
pm_runtime_mark_last_busy(ptdev->base.dev);
pm_runtime_put_autosuspend(ptdev->base.dev);
return done_fence;
}
static enum drm_gpu_sched_stat
queue_timedout_job(struct drm_sched_job *sched_job)
{
struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
struct panthor_group *group = job->group;
struct panthor_device *ptdev = group->ptdev;
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_queue *queue = group->queues[job->queue_idx];
drm_warn(&ptdev->base, "job timeout\n");
drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress));
queue_stop(queue, job);
mutex_lock(&sched->lock);
group->timedout = true;
if (group->csg_id >= 0) {
sched_queue_delayed_work(ptdev->scheduler, tick, 0);
} else {
/* Remove from the run queues, so the scheduler can't
* pick the group on the next tick.
*/
list_del_init(&group->run_node);
list_del_init(&group->wait_node);
group_queue_work(group, term);
}
mutex_unlock(&sched->lock);
queue_start(queue);
return DRM_GPU_SCHED_STAT_NOMINAL;
}
static void queue_free_job(struct drm_sched_job *sched_job)
{
drm_sched_job_cleanup(sched_job);
panthor_job_put(sched_job);
}
static const struct drm_sched_backend_ops panthor_queue_sched_ops = {
.run_job = queue_run_job,
.timedout_job = queue_timedout_job,
.free_job = queue_free_job,
};
static struct panthor_queue *
group_create_queue(struct panthor_group *group,
const struct drm_panthor_queue_create *args)
{
struct drm_gpu_scheduler *drm_sched;
struct panthor_queue *queue;
int ret;
if (args->pad[0] || args->pad[1] || args->pad[2])
return ERR_PTR(-EINVAL);
if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K ||
!is_power_of_2(args->ringbuf_size))
return ERR_PTR(-EINVAL);
if (args->priority > CSF_MAX_QUEUE_PRIO)
return ERR_PTR(-EINVAL);
queue = kzalloc(sizeof(*queue), GFP_KERNEL);
if (!queue)
return ERR_PTR(-ENOMEM);
queue->fence_ctx.id = dma_fence_context_alloc(1);
spin_lock_init(&queue->fence_ctx.lock);
INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs);
queue->priority = args->priority;
queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm,
args->ringbuf_size,
DRM_PANTHOR_BO_NO_MMAP,
DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
PANTHOR_VM_KERNEL_AUTO_VA);
if (IS_ERR(queue->ringbuf)) {
ret = PTR_ERR(queue->ringbuf);
goto err_free_queue;
}
ret = panthor_kernel_bo_vmap(queue->ringbuf);
if (ret)
goto err_free_queue;
queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev,
&queue->iface.input,
&queue->iface.output,
&queue->iface.input_fw_va,
&queue->iface.output_fw_va);
if (IS_ERR(queue->iface.mem)) {
ret = PTR_ERR(queue->iface.mem);
goto err_free_queue;
}
ret = drm_sched_init(&queue->scheduler, &panthor_queue_sched_ops,
group->ptdev->scheduler->wq, 1,
args->ringbuf_size / (NUM_INSTRS_PER_SLOT * sizeof(u64)),
0, msecs_to_jiffies(JOB_TIMEOUT_MS),
group->ptdev->reset.wq,
NULL, "panthor-queue", group->ptdev->base.dev);
if (ret)
goto err_free_queue;
drm_sched = &queue->scheduler;
ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL);
return queue;
err_free_queue:
group_free_queue(group, queue);
return ERR_PTR(ret);
}
#define MAX_GROUPS_PER_POOL 128
int panthor_group_create(struct panthor_file *pfile,
const struct drm_panthor_group_create *group_args,
const struct drm_panthor_queue_create *queue_args)
{
struct panthor_device *ptdev = pfile->ptdev;
struct panthor_group_pool *gpool = pfile->groups;
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
struct panthor_group *group = NULL;
u32 gid, i, suspend_size;
int ret;
if (group_args->pad)
return -EINVAL;
if (group_args->priority >= PANTHOR_CSG_PRIORITY_COUNT)
return -EINVAL;
if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) ||
(group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) ||
(group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present))
return -EINVAL;
if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores ||
hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores ||
hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores)
return -EINVAL;
group = kzalloc(sizeof(*group), GFP_KERNEL);
if (!group)
return -ENOMEM;
spin_lock_init(&group->fatal_lock);
kref_init(&group->refcount);
group->state = PANTHOR_CS_GROUP_CREATED;
group->csg_id = -1;
group->ptdev = ptdev;
group->max_compute_cores = group_args->max_compute_cores;
group->compute_core_mask = group_args->compute_core_mask;
group->max_fragment_cores = group_args->max_fragment_cores;
group->fragment_core_mask = group_args->fragment_core_mask;
group->max_tiler_cores = group_args->max_tiler_cores;
group->tiler_core_mask = group_args->tiler_core_mask;
group->priority = group_args->priority;
INIT_LIST_HEAD(&group->wait_node);
INIT_LIST_HEAD(&group->run_node);
INIT_WORK(&group->term_work, group_term_work);
INIT_WORK(&group->sync_upd_work, group_sync_upd_work);
INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work);
INIT_WORK(&group->release_work, group_release_work);
group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id);
if (!group->vm) {
ret = -EINVAL;
goto err_put_group;
}
suspend_size = csg_iface->control->suspend_size;
group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
if (IS_ERR(group->suspend_buf)) {
ret = PTR_ERR(group->suspend_buf);
group->suspend_buf = NULL;
goto err_put_group;
}
suspend_size = csg_iface->control->protm_suspend_size;
group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
if (IS_ERR(group->protm_suspend_buf)) {
ret = PTR_ERR(group->protm_suspend_buf);
group->protm_suspend_buf = NULL;
goto err_put_group;
}
group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm,
group_args->queues.count *
sizeof(struct panthor_syncobj_64b),
DRM_PANTHOR_BO_NO_MMAP,
DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
PANTHOR_VM_KERNEL_AUTO_VA);
if (IS_ERR(group->syncobjs)) {
ret = PTR_ERR(group->syncobjs);
goto err_put_group;
}
ret = panthor_kernel_bo_vmap(group->syncobjs);
if (ret)
goto err_put_group;
memset(group->syncobjs->kmap, 0,
group_args->queues.count * sizeof(struct panthor_syncobj_64b));
for (i = 0; i < group_args->queues.count; i++) {
group->queues[i] = group_create_queue(group, &queue_args[i]);
if (IS_ERR(group->queues[i])) {
ret = PTR_ERR(group->queues[i]);
group->queues[i] = NULL;
goto err_put_group;
}
group->queue_count++;
}
group->idle_queues = GENMASK(group->queue_count - 1, 0);
ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL);
if (ret)
goto err_put_group;
mutex_lock(&sched->reset.lock);
if (atomic_read(&sched->reset.in_progress)) {
panthor_group_stop(group);
} else {
mutex_lock(&sched->lock);
list_add_tail(&group->run_node,
&sched->groups.idle[group->priority]);
mutex_unlock(&sched->lock);
}
mutex_unlock(&sched->reset.lock);
return gid;
err_put_group:
group_put(group);
return ret;
}
int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle)
{
struct panthor_group_pool *gpool = pfile->groups;
struct panthor_device *ptdev = pfile->ptdev;
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_group *group;
group = xa_erase(&gpool->xa, group_handle);
if (!group)
return -EINVAL;
for (u32 i = 0; i < group->queue_count; i++) {
if (group->queues[i])
drm_sched_entity_destroy(&group->queues[i]->entity);
}
mutex_lock(&sched->reset.lock);
mutex_lock(&sched->lock);
group->destroyed = true;
if (group->csg_id >= 0) {
sched_queue_delayed_work(sched, tick, 0);
} else if (!atomic_read(&sched->reset.in_progress)) {
/* Remove from the run queues, so the scheduler can't
* pick the group on the next tick.
*/
list_del_init(&group->run_node);
list_del_init(&group->wait_node);
group_queue_work(group, term);
}
mutex_unlock(&sched->lock);
mutex_unlock(&sched->reset.lock);
group_put(group);
return 0;
}
static struct panthor_group *group_from_handle(struct panthor_group_pool *pool,
u32 group_handle)
{
struct panthor_group *group;
xa_lock(&pool->xa);
group = group_get(xa_load(&pool->xa, group_handle));
xa_unlock(&pool->xa);
return group;
}
int panthor_group_get_state(struct panthor_file *pfile,
struct drm_panthor_group_get_state *get_state)
{
struct panthor_group_pool *gpool = pfile->groups;
struct panthor_device *ptdev = pfile->ptdev;
struct panthor_scheduler *sched = ptdev->scheduler;
struct panthor_group *group;
if (get_state->pad)
return -EINVAL;
group = group_from_handle(gpool, get_state->group_handle);
if (!group)
return -EINVAL;
memset(get_state, 0, sizeof(*get_state));
mutex_lock(&sched->lock);
if (group->timedout)
get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT;
if (group->fatal_queues) {
get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT;
get_state->fatal_queues = group->fatal_queues;
}
mutex_unlock(&sched->lock);
group_put(group);
return 0;
}
int panthor_group_pool_create(struct panthor_file *pfile)
{
struct panthor_group_pool *gpool;
gpool = kzalloc(sizeof(*gpool), GFP_KERNEL);
if (!gpool)
return -ENOMEM;
xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1);
pfile->groups = gpool;
return 0;
}
void panthor_group_pool_destroy(struct panthor_file *pfile)
{
struct panthor_group_pool *gpool = pfile->groups;
struct panthor_group *group;
unsigned long i;
if (IS_ERR_OR_NULL(gpool))
return;
xa_for_each(&gpool->xa, i, group)
panthor_group_destroy(pfile, i);
xa_destroy(&gpool->xa);
kfree(gpool);
pfile->groups = NULL;
}
static void job_release(struct kref *ref)
{
struct panthor_job *job = container_of(ref, struct panthor_job, refcount);
drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node));
if (job->base.s_fence)
drm_sched_job_cleanup(&job->base);
if (job->done_fence && job->done_fence->ops)
dma_fence_put(job->done_fence);
else
dma_fence_free(job->done_fence);
group_put(job->group);
kfree(job);
}
struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job)
{
if (sched_job) {
struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
kref_get(&job->refcount);
}
return sched_job;
}
void panthor_job_put(struct drm_sched_job *sched_job)
{
struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
if (sched_job)
kref_put(&job->refcount, job_release);
}
struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job)
{
struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
return job->group->vm;
}
struct drm_sched_job *
panthor_job_create(struct panthor_file *pfile,
u16 group_handle,
const struct drm_panthor_queue_submit *qsubmit)
{
struct panthor_group_pool *gpool = pfile->groups;
struct panthor_job *job;
int ret;
if (qsubmit->pad)
return ERR_PTR(-EINVAL);
/* If stream_addr is zero, so stream_size should be. */
if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0))
return ERR_PTR(-EINVAL);
/* Make sure the address is aligned on 64-byte (cacheline) and the size is
* aligned on 8-byte (instruction size).
*/
if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7))
return ERR_PTR(-EINVAL);
/* bits 24:30 must be zero. */
if (qsubmit->latest_flush & GENMASK(30, 24))
return ERR_PTR(-EINVAL);
job = kzalloc(sizeof(*job), GFP_KERNEL);
if (!job)
return ERR_PTR(-ENOMEM);
kref_init(&job->refcount);
job->queue_idx = qsubmit->queue_index;
job->call_info.size = qsubmit->stream_size;
job->call_info.start = qsubmit->stream_addr;
job->call_info.latest_flush = qsubmit->latest_flush;
INIT_LIST_HEAD(&job->node);
job->group = group_from_handle(gpool, group_handle);
if (!job->group) {
ret = -EINVAL;
goto err_put_job;
}
if (job->queue_idx >= job->group->queue_count ||
!job->group->queues[job->queue_idx]) {
ret = -EINVAL;
goto err_put_job;
}
/* Empty command streams don't need a fence, they'll pick the one from
* the previously submitted job.
*/
if (job->call_info.size) {
job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL);
if (!job->done_fence) {
ret = -ENOMEM;
goto err_put_job;
}
}
ret = drm_sched_job_init(&job->base,
&job->group->queues[job->queue_idx]->entity,
1, job->group);
if (ret)
goto err_put_job;
return &job->base;
err_put_job:
panthor_job_put(&job->base);
return ERR_PTR(ret);
}
void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job)
{
struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished,
DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
}
void panthor_sched_unplug(struct panthor_device *ptdev)
{
struct panthor_scheduler *sched = ptdev->scheduler;
cancel_delayed_work_sync(&sched->tick_work);
mutex_lock(&sched->lock);
if (sched->pm.has_ref) {
pm_runtime_put(ptdev->base.dev);
sched->pm.has_ref = false;
}
mutex_unlock(&sched->lock);
}
static void panthor_sched_fini(struct drm_device *ddev, void *res)
{
struct panthor_scheduler *sched = res;
int prio;
if (!sched || !sched->csg_slot_count)
return;
cancel_delayed_work_sync(&sched->tick_work);
if (sched->wq)
destroy_workqueue(sched->wq);
if (sched->heap_alloc_wq)
destroy_workqueue(sched->heap_alloc_wq);
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio]));
drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio]));
}
drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting));
}
int panthor_sched_init(struct panthor_device *ptdev)
{
struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0);
struct panthor_scheduler *sched;
u32 gpu_as_count, num_groups;
int prio, ret;
sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL);
if (!sched)
return -ENOMEM;
/* The highest bit in JOB_INT_* is reserved for globabl IRQs. That
* leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here.
*/
num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num);
/* The FW-side scheduler might deadlock if two groups with the same
* priority try to access a set of resources that overlaps, with part
* of the resources being allocated to one group and the other part to
* the other group, both groups waiting for the remaining resources to
* be allocated. To avoid that, it is recommended to assign each CSG a
* different priority. In theory we could allow several groups to have
* the same CSG priority if they don't request the same resources, but
* that makes the scheduling logic more complicated, so let's clamp
* the number of CSG slots to MAX_CSG_PRIO + 1 for now.
*/
num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups);
/* We need at least one AS for the MCU and one for the GPU contexts. */
gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1));
if (!gpu_as_count) {
drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)",
gpu_as_count + 1);
return -EINVAL;
}
sched->ptdev = ptdev;
sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features);
sched->csg_slot_count = num_groups;
sched->cs_slot_count = csg_iface->control->stream_num;
sched->as_slot_count = gpu_as_count;
ptdev->csif_info.csg_slot_count = sched->csg_slot_count;
ptdev->csif_info.cs_slot_count = sched->cs_slot_count;
ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count;
sched->last_tick = 0;
sched->resched_target = U64_MAX;
sched->tick_period = msecs_to_jiffies(10);
INIT_DELAYED_WORK(&sched->tick_work, tick_work);
INIT_WORK(&sched->sync_upd_work, sync_upd_work);
INIT_WORK(&sched->fw_events_work, process_fw_events_work);
ret = drmm_mutex_init(&ptdev->base, &sched->lock);
if (ret)
return ret;
for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
INIT_LIST_HEAD(&sched->groups.runnable[prio]);
INIT_LIST_HEAD(&sched->groups.idle[prio]);
}
INIT_LIST_HEAD(&sched->groups.waiting);
ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock);
if (ret)
return ret;
INIT_LIST_HEAD(&sched->reset.stopped_groups);
/* sched->heap_alloc_wq will be used for heap chunk allocation on
* tiler OOM events, which means we can't use the same workqueue for
* the scheduler because works queued by the scheduler are in
* the dma-signalling path. Allocate a dedicated heap_alloc_wq to
* work around this limitation.
*
* FIXME: Ultimately, what we need is a failable/non-blocking GEM
* allocation path that we can call when a heap OOM is reported. The
* FW is smart enough to fall back on other methods if the kernel can't
* allocate memory, and fail the tiling job if none of these
* countermeasures worked.
*
* Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the
* system is running out of memory.
*/
sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0);
sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
if (!sched->wq || !sched->heap_alloc_wq) {
panthor_sched_fini(&ptdev->base, sched);
drm_err(&ptdev->base, "Failed to allocate the workqueues");
return -ENOMEM;
}
ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched);
if (ret)
return ret;
ptdev->scheduler = sched;
return 0;
}
Codebase Browser
linux