/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * */ /* * Authors: * Jerome Glisse <[email protected]> * Dave Airlie */ #include <linux/seq_file.h> #include <linux/atomic.h> #include <linux/wait.h> #include <linux/kref.h> #include <linux/slab.h> #include <linux/firmware.h> #include <linux/pm_runtime.h> #include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_trace.h" #include "amdgpu_reset.h" /* * Fences mark an event in the GPUs pipeline and are used * for GPU/CPU synchronization. When the fence is written, * it is expected that all buffers associated with that fence * are no longer in use by the associated ring on the GPU and * that the relevant GPU caches have been flushed. */ struct amdgpu_fence { … }; static struct kmem_cache *amdgpu_fence_slab; int amdgpu_fence_slab_init(void) { … } void amdgpu_fence_slab_fini(void) { … } /* * Cast helper */ static const struct dma_fence_ops amdgpu_fence_ops; static const struct dma_fence_ops amdgpu_job_fence_ops; static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f) { … } /** * amdgpu_fence_write - write a fence value * * @ring: ring the fence is associated with * @seq: sequence number to write * * Writes a fence value to memory (all asics). */ static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq) { … } /** * amdgpu_fence_read - read a fence value * * @ring: ring the fence is associated with * * Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */ static u32 amdgpu_fence_read(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_emit - emit a fence on the requested ring * * @ring: ring the fence is associated with * @f: resulting fence object * @job: job the fence is embedded in * @flags: flags to pass into the subordinate .emit_fence() call * * Emits a fence command on the requested ring (all asics). * Returns 0 on success, -ENOMEM on failure. */ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job, unsigned int flags) { … } /** * amdgpu_fence_emit_polling - emit a fence on the requeste ring * * @ring: ring the fence is associated with * @s: resulting sequence number * @timeout: the timeout for waiting in usecs * * Emits a fence command on the requested ring (all asics). * Used For polling fence. * Returns 0 on success, -ENOMEM on failure. */ int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s, uint32_t timeout) { … } /** * amdgpu_fence_schedule_fallback - schedule fallback check * * @ring: pointer to struct amdgpu_ring * * Start a timer as fallback to our interrupts. */ static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_process - check for fence activity * * @ring: pointer to struct amdgpu_ring * * Checks the current fence value and calculates the last * signalled fence value. Wakes the fence queue if the * sequence number has increased. * * Returns true if fence was processed */ bool amdgpu_fence_process(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_fallback - fallback for hardware interrupts * * @t: timer context used to obtain the pointer to ring structure * * Checks for fence activity. */ static void amdgpu_fence_fallback(struct timer_list *t) { … } /** * amdgpu_fence_wait_empty - wait for all fences to signal * * @ring: ring index the fence is associated with * * Wait for all fences on the requested ring to signal (all asics). * Returns 0 if the fences have passed, error for all other cases. */ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_wait_polling - busy wait for givn sequence number * * @ring: ring index the fence is associated with * @wait_seq: sequence number to wait * @timeout: the timeout for waiting in usecs * * Wait for all fences on the requested ring to signal (all asics). * Returns left time if no timeout, 0 or minus if timeout. */ signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring, uint32_t wait_seq, signed long timeout) { … } /** * amdgpu_fence_count_emitted - get the count of emitted fences * * @ring: ring the fence is associated with * * Get the number of fences emitted on the requested ring (all asics). * Returns the number of emitted fences on the ring. Used by the * dynpm code to ring track activity. */ unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now * @ring: ring the fence is associated with * * Find the earliest fence unsignaled until now, calculate the time delta * between the time fence emitted and now. */ u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_update_start_timestamp - update the timestamp of the fence * @ring: ring the fence is associated with * @seq: the fence seq number to update. * @timestamp: the start timestamp to update. * * The function called at the time the fence and related ib is about to * resubmit to gpu in MCBP scenario. Thus we do not consider race condition * with amdgpu_fence_process to modify the same fence. */ void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp) { … } /** * amdgpu_fence_driver_start_ring - make the fence driver * ready for use on the requested ring. * * @ring: ring to start the fence driver on * @irq_src: interrupt source to use for this ring * @irq_type: interrupt type to use for this ring * * Make the fence driver ready for processing (all asics). * Not all asics have all rings, so each asic will only * start the fence driver on the rings it has. * Returns 0 for success, errors for failure. */ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, struct amdgpu_irq_src *irq_src, unsigned int irq_type) { … } /** * amdgpu_fence_driver_init_ring - init the fence driver * for the requested ring. * * @ring: ring to init the fence driver on * * Init the fence driver for the requested ring (all asics). * Helper function for amdgpu_fence_driver_init(). */ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_driver_sw_init - init the fence driver * for all possible rings. * * @adev: amdgpu device pointer * * Init the fence driver for all possible rings (all asics). * Not all asics have all rings, so each asic will only * start the fence driver on the rings it has using * amdgpu_fence_driver_start_ring(). * Returns 0 for success. */ int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev) { … } /** * amdgpu_fence_need_ring_interrupt_restore - helper function to check whether * fence driver interrupts need to be restored. * * @ring: ring that to be checked * * Interrupts for rings that belong to GFX IP don't need to be restored * when the target power state is s0ix. * * Return true if need to restore interrupts, false otherwise. */ static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_driver_hw_fini - tear down the fence driver * for all possible rings. * * @adev: amdgpu device pointer * * Tear down the fence driver for all possible rings (all asics). */ void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev) { … } /* Will either stop and flush handlers for amdgpu interrupt or reanble it */ void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop) { … } void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev) { … } /** * amdgpu_fence_driver_hw_init - enable the fence driver * for all possible rings. * * @adev: amdgpu device pointer * * Enable the fence driver for all possible rings (all asics). * Not all asics have all rings, so each asic will only * start the fence driver on the rings it has using * amdgpu_fence_driver_start_ring(). * Returns 0 for success. */ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev) { … } /** * amdgpu_fence_driver_clear_job_fences - clear job embedded fences of ring * * @ring: fence of the ring to be cleared * */ void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring) { … } /** * amdgpu_fence_driver_set_error - set error code on fences * @ring: the ring which contains the fences * @error: the error code to set * * Set an error code to all the fences pending on the ring. */ void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error) { … } /** * amdgpu_fence_driver_force_completion - force signal latest fence of ring * * @ring: fence of the ring to signal * */ void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring) { … } /* * Common fence implementation */ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence) { … } static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f) { … } static const char *amdgpu_job_fence_get_timeline_name(struct dma_fence *f) { … } /** * amdgpu_fence_enable_signaling - enable signalling on fence * @f: fence * * This function is called with fence_queue lock held, and adds a callback * to fence_queue that checks if this fence is signaled, and if so it * signals the fence and removes itself. */ static bool amdgpu_fence_enable_signaling(struct dma_fence *f) { … } /** * amdgpu_job_fence_enable_signaling - enable signalling on job fence * @f: fence * * This is the simliar function with amdgpu_fence_enable_signaling above, it * only handles the job embedded fence. */ static bool amdgpu_job_fence_enable_signaling(struct dma_fence *f) { … } /** * amdgpu_fence_free - free up the fence memory * * @rcu: RCU callback head * * Free up the fence memory after the RCU grace period. */ static void amdgpu_fence_free(struct rcu_head *rcu) { … } /** * amdgpu_job_fence_free - free up the job with embedded fence * * @rcu: RCU callback head * * Free up the job with embedded fence after the RCU grace period. */ static void amdgpu_job_fence_free(struct rcu_head *rcu) { … } /** * amdgpu_fence_release - callback that fence can be freed * * @f: fence * * This function is called when the reference count becomes zero. * It just RCU schedules freeing up the fence. */ static void amdgpu_fence_release(struct dma_fence *f) { … } /** * amdgpu_job_fence_release - callback that job embedded fence can be freed * * @f: fence * * This is the simliar function with amdgpu_fence_release above, it * only handles the job embedded fence. */ static void amdgpu_job_fence_release(struct dma_fence *f) { … } static const struct dma_fence_ops amdgpu_fence_ops = …; static const struct dma_fence_ops amdgpu_job_fence_ops = …; /* * Fence debugfs */ #if defined(CONFIG_DEBUG_FS) static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused) { … } /* * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover * * Manually trigger a gpu reset at the next fence wait. */ static int gpu_recover_get(void *data, u64 *val) { … } DEFINE_SHOW_ATTRIBUTE(…); DEFINE_DEBUGFS_ATTRIBUTE(…); static void amdgpu_debugfs_reset_work(struct work_struct *work) { … } #endif void amdgpu_debugfs_fence_init(struct amdgpu_device *adev) { … }
Codebase Browser
linux