/****************************************************************************** * grant_table.c * * Granting foreign access to our memory reservation. * * Copyright (c) 2005-2006, Christopher Clark * Copyright (c) 2004-2005, K A Fraser * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 2 * as published by the Free Software Foundation; or, when distributed * separately from the Linux kernel or incorporated into other * software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #define pr_fmt(fmt) … #include <linux/bitmap.h> #include <linux/memblock.h> #include <linux/sched.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/uaccess.h> #include <linux/io.h> #include <linux/delay.h> #include <linux/hardirq.h> #include <linux/workqueue.h> #include <linux/ratelimit.h> #include <linux/moduleparam.h> #ifdef CONFIG_XEN_GRANT_DMA_ALLOC #include <linux/dma-mapping.h> #endif #include <xen/xen.h> #include <xen/interface/xen.h> #include <xen/page.h> #include <xen/grant_table.h> #include <xen/interface/memory.h> #include <xen/hvc-console.h> #include <xen/swiotlb-xen.h> #include <xen/balloon.h> #ifdef CONFIG_X86 #include <asm/xen/cpuid.h> #endif #include <xen/mem-reservation.h> #include <asm/xen/hypercall.h> #include <asm/xen/interface.h> #include <asm/sync_bitops.h> #define GNTTAB_LIST_END … static grant_ref_t **gnttab_list; static unsigned int nr_grant_frames; /* * Handling of free grants: * * Free grants are in a simple list anchored in gnttab_free_head. They are * linked by grant ref, the last element contains GNTTAB_LIST_END. The number * of free entries is stored in gnttab_free_count. * Additionally there is a bitmap of free entries anchored in * gnttab_free_bitmap. This is being used for simplifying allocation of * multiple consecutive grants, which is needed e.g. for support of virtio. * gnttab_last_free is used to add free entries of new frames at the end of * the free list. * gnttab_free_tail_ptr specifies the variable which references the start * of consecutive free grants ending with gnttab_last_free. This pointer is * updated in a rather defensive way, in order to avoid performance hits in * hot paths. * All those variables are protected by gnttab_list_lock. */ static int gnttab_free_count; static unsigned int gnttab_size; static grant_ref_t gnttab_free_head = …; static grant_ref_t gnttab_last_free = …; static grant_ref_t *gnttab_free_tail_ptr; static unsigned long *gnttab_free_bitmap; static DEFINE_SPINLOCK(gnttab_list_lock); struct grant_frames xen_auto_xlat_grant_frames; static unsigned int xen_gnttab_version; module_param_named(version, xen_gnttab_version, uint, 0); static union { … } gnttab_shared; /*This is a structure of function pointers for grant table*/ struct gnttab_ops { … }; struct unmap_refs_callback_data { … }; static const struct gnttab_ops *gnttab_interface; /* This reflects status of grant entries, so act as a global value. */ static grant_status_t *grstatus; static struct gnttab_free_callback *gnttab_free_callback_list; static int gnttab_expand(unsigned int req_entries); #define RPP … #define SPP … static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) { … } /* This can be used as an l-value */ #define gnttab_entry(entry) … static int get_free_entries(unsigned count) { … } static int get_seq_entry_count(void) { … } /* Rebuilds the free grant list and tries to find count consecutive entries. */ static int get_free_seq(unsigned int count) { … } static int get_free_entries_seq(unsigned int count) { … } static void do_free_callbacks(void) { … } static inline void check_free_callbacks(void) { … } static void put_free_entry_locked(grant_ref_t ref) { … } static void put_free_entry(grant_ref_t ref) { … } static void gnttab_set_free(unsigned int start, unsigned int n) { … } /* * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2. * Introducing a valid entry into the grant table: * 1. Write ent->domid. * 2. Write ent->frame: Frame to which access is permitted. * 3. Write memory barrier (WMB). * 4. Write ent->flags, inc. valid type. */ static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid, unsigned long frame, unsigned flags) { … } static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid, unsigned long frame, unsigned int flags) { … } /* * Public grant-issuing interface functions */ void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, unsigned long frame, int readonly) { … } EXPORT_SYMBOL_GPL(…); int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly) { … } EXPORT_SYMBOL_GPL(…); static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref) { … } static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref) { … } static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref) { … } int gnttab_end_foreign_access_ref(grant_ref_t ref) { … } EXPORT_SYMBOL_GPL(…); static unsigned long gnttab_read_frame_v1(grant_ref_t ref) { … } static unsigned long gnttab_read_frame_v2(grant_ref_t ref) { … } struct deferred_entry { … }; static LIST_HEAD(deferred_list); static void gnttab_handle_deferred(struct timer_list *); static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred); static atomic64_t deferred_count; static atomic64_t leaked_count; static unsigned int free_per_iteration = …; module_param(free_per_iteration, uint, 0600); static void gnttab_handle_deferred(struct timer_list *unused) { … } static void gnttab_add_deferred(grant_ref_t ref, struct page *page) { … } int gnttab_try_end_foreign_access(grant_ref_t ref) { … } EXPORT_SYMBOL_GPL(…); void gnttab_end_foreign_access(grant_ref_t ref, struct page *page) { … } EXPORT_SYMBOL_GPL(…); void gnttab_free_grant_reference(grant_ref_t ref) { … } EXPORT_SYMBOL_GPL(…); void gnttab_free_grant_references(grant_ref_t head) { … } EXPORT_SYMBOL_GPL(…); void gnttab_free_grant_reference_seq(grant_ref_t head, unsigned int count) { … } EXPORT_SYMBOL_GPL(…); int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) { … } EXPORT_SYMBOL_GPL(…); int gnttab_alloc_grant_reference_seq(unsigned int count, grant_ref_t *first) { … } EXPORT_SYMBOL_GPL(…); int gnttab_empty_grant_references(const grant_ref_t *private_head) { … } EXPORT_SYMBOL_GPL(…); int gnttab_claim_grant_reference(grant_ref_t *private_head) { … } EXPORT_SYMBOL_GPL(…); void gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) { … } EXPORT_SYMBOL_GPL(…); void gnttab_request_free_callback(struct gnttab_free_callback *callback, void (*fn)(void *), void *arg, u16 count) { … } EXPORT_SYMBOL_GPL(…); void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) { … } EXPORT_SYMBOL_GPL(…); static unsigned int gnttab_frames(unsigned int frames, unsigned int align) { … } static int grow_gnttab_list(unsigned int more_frames) { … } static unsigned int __max_nr_grant_frames(void) { … } unsigned int gnttab_max_grant_frames(void) { … } EXPORT_SYMBOL_GPL(…); int gnttab_setup_auto_xlat_frames(phys_addr_t addr) { … } EXPORT_SYMBOL_GPL(…); void gnttab_free_auto_xlat_frames(void) { … } EXPORT_SYMBOL_GPL(…); int gnttab_pages_set_private(int nr_pages, struct page **pages) { … } EXPORT_SYMBOL_GPL(…); /** * gnttab_alloc_pages - alloc pages suitable for grant mapping into * @nr_pages: number of pages to alloc * @pages: returns the pages */ int gnttab_alloc_pages(int nr_pages, struct page **pages) { … } EXPORT_SYMBOL_GPL(…); #ifdef CONFIG_XEN_UNPOPULATED_ALLOC static inline void cache_init(struct gnttab_page_cache *cache) { … } static inline bool cache_empty(struct gnttab_page_cache *cache) { … } static inline struct page *cache_deq(struct gnttab_page_cache *cache) { … } static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page) { … } #else static inline void cache_init(struct gnttab_page_cache *cache) { INIT_LIST_HEAD(&cache->pages); } static inline bool cache_empty(struct gnttab_page_cache *cache) { return list_empty(&cache->pages); } static inline struct page *cache_deq(struct gnttab_page_cache *cache) { struct page *page; page = list_first_entry(&cache->pages, struct page, lru); list_del(&page->lru); return page; } static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page) { list_add(&page->lru, &cache->pages); } #endif void gnttab_page_cache_init(struct gnttab_page_cache *cache) { … } EXPORT_SYMBOL_GPL(…); int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page) { … } EXPORT_SYMBOL_GPL(…); void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page, unsigned int num) { … } EXPORT_SYMBOL_GPL(…); void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num) { … } EXPORT_SYMBOL_GPL(…); void gnttab_pages_clear_private(int nr_pages, struct page **pages) { … } EXPORT_SYMBOL_GPL(…); /** * gnttab_free_pages - free pages allocated by gnttab_alloc_pages() * @nr_pages: number of pages to free * @pages: the pages */ void gnttab_free_pages(int nr_pages, struct page **pages) { … } EXPORT_SYMBOL_GPL(…); #ifdef CONFIG_XEN_GRANT_DMA_ALLOC /** * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into * @args: arguments to the function */ int gnttab_dma_alloc_pages(struct gnttab_dma_alloc_args *args) { … } EXPORT_SYMBOL_GPL(…); /** * gnttab_dma_free_pages - free DMAable pages * @args: arguments to the function */ int gnttab_dma_free_pages(struct gnttab_dma_alloc_args *args) { … } EXPORT_SYMBOL_GPL(…); #endif /* Handling of paged out grant targets (GNTST_eagain) */ #define MAX_DELAY … static inline void gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status, const char *func) { … } void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count) { … } EXPORT_SYMBOL_GPL(…); void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count) { … } EXPORT_SYMBOL_GPL(…); void gnttab_foreach_grant_in_range(struct page *page, unsigned int offset, unsigned int len, xen_grant_fn_t fn, void *data) { … } EXPORT_SYMBOL_GPL(…); void gnttab_foreach_grant(struct page **pages, unsigned int nr_grefs, xen_grant_fn_t fn, void *data) { … } int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, struct gnttab_map_grant_ref *kmap_ops, struct page **pages, unsigned int count) { … } EXPORT_SYMBOL_GPL(…); int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, struct gnttab_unmap_grant_ref *kunmap_ops, struct page **pages, unsigned int count) { … } EXPORT_SYMBOL_GPL(…); #define GNTTAB_UNMAP_REFS_DELAY … static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item); static void gnttab_unmap_work(struct work_struct *work) { … } static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) { … } void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) { … } EXPORT_SYMBOL_GPL(…); static void unmap_refs_callback(int result, struct gntab_unmap_queue_data *data) { … } int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item) { … } EXPORT_SYMBOL_GPL(…); static unsigned int nr_status_frames(unsigned int nr_grant_frames) { … } static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes) { … } static void gnttab_unmap_frames_v1(void) { … } static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes) { … } static void gnttab_unmap_frames_v2(void) { … } static int gnttab_map(unsigned int start_idx, unsigned int end_idx) { … } static const struct gnttab_ops gnttab_v1_ops = …; static const struct gnttab_ops gnttab_v2_ops = …; static bool gnttab_need_v2(void) { … } static void gnttab_request_version(void) { … } static int gnttab_setup(void) { … } int gnttab_resume(void) { … } int gnttab_suspend(void) { … } static int gnttab_expand(unsigned int req_entries) { … } int gnttab_init(void) { … } EXPORT_SYMBOL_GPL(…); static int __gnttab_init(void) { … } /* Starts after core_initcall so that xen_pvh_gnttab_setup can be called * beforehand to initialize xen_auto_xlat_grant_frames. */ core_initcall_sync(__gnttab_init);
Codebase Browser
linux