/* * Copyright © 2017 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * */ #include <linux/highmem.h> #include <linux/sched/mm.h> #include <drm/drm_cache.h> #include "display/intel_frontbuffer.h" #include "pxp/intel_pxp.h" #include "i915_drv.h" #include "i915_file_private.h" #include "i915_gem_clflush.h" #include "i915_gem_context.h" #include "i915_gem_dmabuf.h" #include "i915_gem_mman.h" #include "i915_gem_object.h" #include "i915_gem_object_frontbuffer.h" #include "i915_gem_ttm.h" #include "i915_memcpy.h" #include "i915_trace.h" static struct kmem_cache *slab_objects; static const struct drm_gem_object_funcs i915_gem_object_funcs; unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915, enum i915_cache_level level) { … } bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj, enum i915_cache_level lvl) { … } struct drm_i915_gem_object *i915_gem_object_alloc(void) { … } void i915_gem_object_free(struct drm_i915_gem_object *obj) { … } void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops, struct lock_class_key *key, unsigned flags) { … } /** * __i915_gem_object_fini - Clean up a GEM object initialization * @obj: The gem object to cleanup * * This function cleans up gem object fields that are set up by * drm_gem_private_object_init() and i915_gem_object_init(). * It's primarily intended as a helper for backends that need to * clean up the gem object in separate steps. */ void __i915_gem_object_fini(struct drm_i915_gem_object *obj) { … } /** * i915_gem_object_set_cache_coherency - Mark up the object's coherency levels * for a given cache_level * @obj: #drm_i915_gem_object * @cache_level: cache level */ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, unsigned int cache_level) { … } /** * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode * @obj: #drm_i915_gem_object * @pat_index: PAT index * * This is a clone of i915_gem_object_set_cache_coherency taking pat index * instead of cache_level as its second argument. */ void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj, unsigned int pat_index) { … } bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) { … } static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) { … } void __i915_gem_free_object_rcu(struct rcu_head *head) { … } static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) { … } /** * __i915_gem_object_pages_fini - Clean up pages use of a gem object * @obj: The gem object to clean up * * This function cleans up usage of the object mm.pages member. It * is intended for backends that need to clean up a gem object in * separate steps and needs to be called when the object is idle before * the object's backing memory is freed. */ void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) { … } void __i915_gem_free_object(struct drm_i915_gem_object *obj) { … } static void __i915_gem_free_objects(struct drm_i915_private *i915, struct llist_node *freed) { … } void i915_gem_flush_free_objects(struct drm_i915_private *i915) { … } static void __i915_gem_free_work(struct work_struct *work) { … } static void i915_gem_free_object(struct drm_gem_object *gem_obj) { … } void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, enum fb_op_origin origin) { … } void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, enum fb_op_origin origin) { … } static void i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) { … } static void i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) { … } static bool object_has_mappable_iomem(struct drm_i915_gem_object *obj) { … } /** * i915_gem_object_read_from_page - read data from the page of a GEM object * @obj: GEM object to read from * @offset: offset within the object * @dst: buffer to store the read data * @size: size to read * * Reads data from @obj at the specified offset. The requested region to read * from can't cross a page boundary. The caller must ensure that @obj pages * are pinned and that @obj is synced wrt. any related writes. * * Return: %0 on success or -ENODEV if the type of @obj's backing store is * unsupported. */ int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) { … } /** * i915_gem_object_evictable - Whether object is likely evictable after unbind. * @obj: The object to check * * This function checks whether the object is likely unvictable after unbind. * If the object is not locked when checking, the result is only advisory. * If the object is locked when checking, and the function returns true, * then an eviction should indeed be possible. But since unlocked vma * unpinning and unbinding is currently possible, the object can actually * become evictable even if this function returns false. * * Return: true if the object may be evictable. False otherwise. */ bool i915_gem_object_evictable(struct drm_i915_gem_object *obj) { … } /** * i915_gem_object_migratable - Whether the object is migratable out of the * current region. * @obj: Pointer to the object. * * Return: Whether the object is allowed to be resident in other * regions than the current while pages are present. */ bool i915_gem_object_migratable(struct drm_i915_gem_object *obj) { … } /** * i915_gem_object_has_struct_page - Whether the object is page-backed * @obj: The object to query. * * This function should only be called while the object is locked or pinned, * otherwise the page backing may change under the caller. * * Return: True if page-backed, false otherwise. */ bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) { … } /** * i915_gem_object_has_iomem - Whether the object is iomem-backed * @obj: The object to query. * * This function should only be called while the object is locked or pinned, * otherwise the iomem backing may change under the caller. * * Return: True if iomem-backed, false otherwise. */ bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj) { … } /** * i915_gem_object_can_migrate - Whether an object likely can be migrated * * @obj: The object to migrate * @id: The region intended to migrate to * * Check whether the object backend supports migration to the * given region. Note that pinning may affect the ability to migrate as * returned by this function. * * This function is primarily intended as a helper for checking the * possibility to migrate objects and might be slightly less permissive * than i915_gem_object_migrate() when it comes to objects with the * I915_BO_ALLOC_USER flag set. * * Return: true if migration is possible, false otherwise. */ bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, enum intel_region_id id) { … } /** * i915_gem_object_migrate - Migrate an object to the desired region id * @obj: The object to migrate. * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may * not be successful in evicting other objects to make room for this object. * @id: The region id to migrate to. * * Attempt to migrate the object to the desired memory region. The * object backend must support migration and the object may not be * pinned, (explicitly pinned pages or pinned vmas). The object must * be locked. * On successful completion, the object will have pages pointing to * memory in the new region, but an async migration task may not have * completed yet, and to accomplish that, i915_gem_object_wait_migration() * must be called. * * Note: the @ww parameter is not used yet, but included to make sure * callers put some effort into obtaining a valid ww ctx if one is * available. * * Return: 0 on success. Negative error code on failure. In particular may * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and * -EBUSY if the object is pinned. */ int i915_gem_object_migrate(struct drm_i915_gem_object *obj, struct i915_gem_ww_ctx *ww, enum intel_region_id id) { … } /** * __i915_gem_object_migrate - Migrate an object to the desired region id, with * control of the extra flags * @obj: The object to migrate. * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may * not be successful in evicting other objects to make room for this object. * @id: The region id to migrate to. * @flags: The object flags. Normally just obj->flags. * * Attempt to migrate the object to the desired memory region. The * object backend must support migration and the object may not be * pinned, (explicitly pinned pages or pinned vmas). The object must * be locked. * On successful completion, the object will have pages pointing to * memory in the new region, but an async migration task may not have * completed yet, and to accomplish that, i915_gem_object_wait_migration() * must be called. * * Note: the @ww parameter is not used yet, but included to make sure * callers put some effort into obtaining a valid ww ctx if one is * available. * * Return: 0 on success. Negative error code on failure. In particular may * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and * -EBUSY if the object is pinned. */ int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, struct i915_gem_ww_ctx *ww, enum intel_region_id id, unsigned int flags) { … } /** * i915_gem_object_placement_possible - Check whether the object can be * placed at certain memory type * @obj: Pointer to the object * @type: The memory type to check * * Return: True if the object can be placed in @type. False otherwise. */ bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, enum intel_memory_type type) { … } /** * i915_gem_object_needs_ccs_pages - Check whether the object requires extra * pages when placed in system-memory, in order to save and later restore the * flat-CCS aux state when the object is moved between local-memory and * system-memory * @obj: Pointer to the object * * Return: True if the object needs extra ccs pages. False otherwise. */ bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj) { … } void i915_gem_init__objects(struct drm_i915_private *i915) { … } void i915_objects_module_exit(void) { … } int __init i915_objects_module_init(void) { … } static const struct drm_gem_object_funcs i915_gem_object_funcs = …; /** * i915_gem_object_get_moving_fence - Get the object's moving fence if any * @obj: The object whose moving fence to get. * @fence: The resulting fence * * A non-signaled moving fence means that there is an async operation * pending on the object that needs to be waited on before setting up * any GPU- or CPU PTEs to the object's pages. * * Return: Negative error code or 0 for success. */ int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, struct dma_fence **fence) { … } /** * i915_gem_object_wait_moving_fence - Wait for the object's moving fence if any * @obj: The object whose moving fence to wait for. * @intr: Whether to wait interruptible. * * If the moving fence signaled without an error, it is detached from the * object and put. * * Return: 0 if successful, -ERESTARTSYS if the wait was interrupted, * negative error code if the async operation represented by the * moving fence failed. */ int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, bool intr) { … } /* * i915_gem_object_has_unknown_state - Return true if the object backing pages are * in an unknown_state. This means that userspace must NEVER be allowed to touch * the pages, with either the GPU or CPU. * * ONLY valid to be called after ensuring that all kernel fences have signalled * (in particular the fence for moving/clearing the object). */ bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj) { … } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/huge_gem_object.c" #include "selftests/huge_pages.c" #include "selftests/i915_gem_migrate.c" #include "selftests/i915_gem_object.c" #include "selftests/i915_gem_coherency.c" #endif
Codebase Browser
linux