// SPDX-License-Identifier: GPL-2.0-only /* * Copyright © 2006-2009, Intel Corporation. * * Author: Anil S Keshavamurthy <[email protected]> */ #include <linux/iova.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/smp.h> #include <linux/bitops.h> #include <linux/cpu.h> #include <linux/workqueue.h> /* The anchor node sits above the top of the usable address space */ #define IOVA_ANCHOR … #define IOVA_RANGE_CACHE_MAX_SIZE … static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, unsigned long size); static unsigned long iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn); static void free_iova_rcaches(struct iova_domain *iovad); static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_global_cached_iovas(struct iova_domain *iovad); static struct iova *to_iova(struct rb_node *node) { … } void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn) { … } EXPORT_SYMBOL_GPL(…); static struct rb_node * __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) { … } static void __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) { … } static void __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) { … } static struct rb_node *iova_find_limit(struct iova_domain *iovad, unsigned long limit_pfn) { … } /* Insert the iova into domain rbtree by holding writer lock */ static void iova_insert_rbtree(struct rb_root *root, struct iova *iova, struct rb_node *start) { … } static int __alloc_and_insert_iova_range(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, struct iova *new, bool size_aligned) { … } static struct kmem_cache *iova_cache; static unsigned int iova_cache_users; static DEFINE_MUTEX(iova_cache_mutex); static struct iova *alloc_iova_mem(void) { … } static void free_iova_mem(struct iova *iova) { … } /** * alloc_iova - allocates an iova * @iovad: - iova domain in question * @size: - size of page frames to allocate * @limit_pfn: - max limit address * @size_aligned: - set if size_aligned address range is required * This function allocates an iova in the range iovad->start_pfn to limit_pfn, * searching top-down from limit_pfn to iovad->start_pfn. If the size_aligned * flag is set then the allocated address iova->pfn_lo will be naturally * aligned on roundup_power_of_two(size). */ struct iova * alloc_iova(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool size_aligned) { … } EXPORT_SYMBOL_GPL(…); static struct iova * private_find_iova(struct iova_domain *iovad, unsigned long pfn) { … } static void remove_iova(struct iova_domain *iovad, struct iova *iova) { … } /** * find_iova - finds an iova for a given pfn * @iovad: - iova domain in question. * @pfn: - page frame number * This function finds and returns an iova belonging to the * given domain which matches the given pfn. */ struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) { … } EXPORT_SYMBOL_GPL(…); /** * __free_iova - frees the given iova * @iovad: iova domain in question. * @iova: iova in question. * Frees the given iova belonging to the giving domain */ void __free_iova(struct iova_domain *iovad, struct iova *iova) { … } EXPORT_SYMBOL_GPL(…); /** * free_iova - finds and frees the iova for a given pfn * @iovad: - iova domain in question. * @pfn: - pfn that is allocated previously * This functions finds an iova for a given pfn and then * frees the iova from that domain. */ void free_iova(struct iova_domain *iovad, unsigned long pfn) { … } EXPORT_SYMBOL_GPL(…); /** * alloc_iova_fast - allocates an iova from rcache * @iovad: - iova domain in question * @size: - size of page frames to allocate * @limit_pfn: - max limit address * @flush_rcache: - set to flush rcache on regular allocation failure * This function tries to satisfy an iova allocation from the rcache, * and falls back to regular allocation on failure. If regular allocation * fails too and the flush_rcache flag is set then the rcache will be flushed. */ unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache) { … } EXPORT_SYMBOL_GPL(…); /** * free_iova_fast - free iova pfn range into rcache * @iovad: - iova domain in question. * @pfn: - pfn that is allocated previously * @size: - # of pages in range * This functions frees an iova range by trying to put it into the rcache, * falling back to regular iova deallocation via free_iova() if this fails. */ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) { … } EXPORT_SYMBOL_GPL(…); static void iova_domain_free_rcaches(struct iova_domain *iovad) { … } /** * put_iova_domain - destroys the iova domain * @iovad: - iova domain in question. * All the iova's in that domain are destroyed. */ void put_iova_domain(struct iova_domain *iovad) { … } EXPORT_SYMBOL_GPL(…); static int __is_range_overlap(struct rb_node *node, unsigned long pfn_lo, unsigned long pfn_hi) { … } static inline struct iova * alloc_and_init_iova(unsigned long pfn_lo, unsigned long pfn_hi) { … } static struct iova * __insert_new_range(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) { … } static void __adjust_overlap_range(struct iova *iova, unsigned long *pfn_lo, unsigned long *pfn_hi) { … } /** * reserve_iova - reserves an iova in the given range * @iovad: - iova domain pointer * @pfn_lo: - lower page frame address * @pfn_hi:- higher pfn adderss * This function allocates reserves the address range from pfn_lo to pfn_hi so * that this address is not dished out as part of alloc_iova. */ struct iova * reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) { … } EXPORT_SYMBOL_GPL(…); /* * Magazine caches for IOVA ranges. For an introduction to magazines, * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab * Allocator to Many CPUs and Arbitrary Resources" by Bonwick and Adams. * For simplicity, we use a static magazine size and don't implement the * dynamic size tuning described in the paper. */ /* * As kmalloc's buffer size is fixed to power of 2, 127 is chosen to * assure size of 'iova_magazine' to be 1024 bytes, so that no memory * will be wasted. Since only full magazines are inserted into the depot, * we don't need to waste PFN capacity on a separate list head either. */ #define IOVA_MAG_SIZE … #define IOVA_DEPOT_DELAY … struct iova_magazine { … }; static_assert(…); struct iova_cpu_rcache { … }; struct iova_rcache { … }; static struct kmem_cache *iova_magazine_cache; unsigned long iova_rcache_range(void) { … } static struct iova_magazine *iova_magazine_alloc(gfp_t flags) { … } static void iova_magazine_free(struct iova_magazine *mag) { … } static void iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) { … } static bool iova_magazine_full(struct iova_magazine *mag) { … } static bool iova_magazine_empty(struct iova_magazine *mag) { … } static unsigned long iova_magazine_pop(struct iova_magazine *mag, unsigned long limit_pfn) { … } static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) { … } static struct iova_magazine *iova_depot_pop(struct iova_rcache *rcache) { … } static void iova_depot_push(struct iova_rcache *rcache, struct iova_magazine *mag) { … } static void iova_depot_work_func(struct work_struct *work) { … } int iova_domain_init_rcaches(struct iova_domain *iovad) { … } EXPORT_SYMBOL_GPL(…); /* * Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and * return true on success. Can fail if rcache is full and we can't free * space, and free_iova() (our only caller) will then return the IOVA * range to the rbtree instead. */ static bool __iova_rcache_insert(struct iova_domain *iovad, struct iova_rcache *rcache, unsigned long iova_pfn) { … } static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, unsigned long size) { … } /* * Caller wants to allocate a new IOVA range from 'rcache'. If we can * satisfy the request, return a matching non-NULL range and remove * it from the 'rcache'. */ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, unsigned long limit_pfn) { … } /* * Try to satisfy IOVA allocation range from rcache. Fail if requested * size is too big or the DMA limit we are given isn't satisfied by the * top element in the magazine. */ static unsigned long iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn) { … } /* * free rcache data structures. */ static void free_iova_rcaches(struct iova_domain *iovad) { … } /* * free all the IOVA ranges cached by a cpu (used when cpu is unplugged) */ static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) { … } /* * free all the IOVA ranges of global cache */ static void free_global_cached_iovas(struct iova_domain *iovad) { … } static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) { … } int iova_cache_get(void) { … } EXPORT_SYMBOL_GPL(…); void iova_cache_put(void) { … } EXPORT_SYMBOL_GPL(…); MODULE_AUTHOR(…) …; MODULE_DESCRIPTION(…) …; MODULE_LICENSE(…) …;
Codebase Browser
linux