// SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2016-20 Intel Corporation. */ #include <linux/file.h> #include <linux/freezer.h> #include <linux/highmem.h> #include <linux/kthread.h> #include <linux/miscdevice.h> #include <linux/node.h> #include <linux/pagemap.h> #include <linux/ratelimit.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> #include <linux/slab.h> #include <linux/sysfs.h> #include <linux/vmalloc.h> #include <asm/sgx.h> #include "driver.h" #include "encl.h" #include "encls.h" struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; static int sgx_nr_epc_sections; static struct task_struct *ksgxd_tsk; static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq); static DEFINE_XARRAY(sgx_epc_address_space); /* * These variables are part of the state of the reclaimer, and must be accessed * with sgx_reclaimer_lock acquired. */ static LIST_HEAD(sgx_active_page_list); static DEFINE_SPINLOCK(sgx_reclaimer_lock); static atomic_long_t sgx_nr_free_pages = …; /* Nodes with one or more EPC sections. */ static nodemask_t sgx_numa_mask; /* * Array with one list_head for each possible NUMA node. Each * list contains all the sgx_epc_section's which are on that * node. */ static struct sgx_numa_node *sgx_numa_nodes; static LIST_HEAD(sgx_dirty_page_list); /* * Reset post-kexec EPC pages to the uninitialized state. The pages are removed * from the input list, and made available for the page allocator. SECS pages * prepending their children in the input list are left intact. * * Return 0 when sanitization was successful or kthread was stopped, and the * number of unsanitized pages otherwise. */ static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list) { … } static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) { … } static void sgx_reclaimer_block(struct sgx_epc_page *epc_page) { … } static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot, struct sgx_backing *backing) { … } void sgx_ipi_cb(void *info) { … } /* * Swap page to the regular memory transformed to the blocked state by using * EBLOCK, which means that it can no longer be referenced (no new TLB entries). * * The first trial just tries to write the page assuming that some other thread * has reset the count for threads inside the enclave by using ETRACK, and * previous thread count has been zeroed out. The second trial calls ETRACK * before EWB. If that fails we kick all the HW threads out, and then do EWB, * which should be guaranteed the succeed. */ static void sgx_encl_ewb(struct sgx_epc_page *epc_page, struct sgx_backing *backing) { … } static void sgx_reclaimer_write(struct sgx_epc_page *epc_page, struct sgx_backing *backing) { … } /* * Take a fixed number of pages from the head of the active page pool and * reclaim them to the enclave's private shmem files. Skip the pages, which have * been accessed since the last scan. Move those pages to the tail of active * page pool so that the pages get scanned in LRU like fashion. * * Batch process a chunk of pages (at the moment 16) in order to degrade amount * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI * + EWB) but not sufficiently. Reclaiming one page at a time would also be * problematic as it would increase the lock contention too much, which would * halt forward progress. */ static void sgx_reclaim_pages(void) { … } static bool sgx_should_reclaim(unsigned long watermark) { … } /* * sgx_reclaim_direct() should be called (without enclave's mutex held) * in locations where SGX memory resources might be low and might be * needed in order to make forward progress. */ void sgx_reclaim_direct(void) { … } static int ksgxd(void *p) { … } static bool __init sgx_page_reclaimer_init(void) { … } bool current_is_ksgxd(void) { … } static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid) { … } /** * __sgx_alloc_epc_page() - Allocate an EPC page * * Iterate through NUMA nodes and reserve ia free EPC page to the caller. Start * from the NUMA node, where the caller is executing. * * Return: * - an EPC page: A borrowed EPC pages were available. * - NULL: Out of EPC pages. */ struct sgx_epc_page *__sgx_alloc_epc_page(void) { … } /** * sgx_mark_page_reclaimable() - Mark a page as reclaimable * @page: EPC page * * Mark a page as reclaimable and add it to the active page list. Pages * are automatically removed from the active list when freed. */ void sgx_mark_page_reclaimable(struct sgx_epc_page *page) { … } /** * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list * @page: EPC page * * Clear the reclaimable flag and remove the page from the active page list. * * Return: * 0 on success, * -EBUSY if the page is in the process of being reclaimed */ int sgx_unmark_page_reclaimable(struct sgx_epc_page *page) { … } /** * sgx_alloc_epc_page() - Allocate an EPC page * @owner: the owner of the EPC page * @reclaim: reclaim pages if necessary * * Iterate through EPC sections and borrow a free EPC page to the caller. When a * page is no longer needed it must be released with sgx_free_epc_page(). If * @reclaim is set to true, directly reclaim pages when we are out of pages. No * mm's can be locked when @reclaim is set to true. * * Finally, wake up ksgxd when the number of pages goes below the watermark * before returning back to the caller. * * Return: * an EPC page, * -errno on error */ struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim) { … } /** * sgx_free_epc_page() - Free an EPC page * @page: an EPC page * * Put the EPC page back to the list of free pages. It's the caller's * responsibility to make sure that the page is in uninitialized state. In other * words, do EREMOVE, EWB or whatever operation is necessary before calling * this function. */ void sgx_free_epc_page(struct sgx_epc_page *page) { … } static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, unsigned long index, struct sgx_epc_section *section) { … } bool arch_is_platform_page(u64 paddr) { … } EXPORT_SYMBOL_GPL(…); static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr) { … } /* * Called in process context to handle a hardware reported * error in an SGX EPC page. * If the MF_ACTION_REQUIRED bit is set in flags, then the * context is the task that consumed the poison data. Otherwise * this is called from a kernel thread unrelated to the page. */ int arch_memory_failure(unsigned long pfn, int flags) { … } /* * A section metric is concatenated in a way that @low bits 12-31 define the * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the * metric. */ static inline u64 __init sgx_calc_section_metric(u64 low, u64 high) { … } #ifdef CONFIG_NUMA static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf) { … } static DEVICE_ATTR_RO(sgx_total_bytes); static umode_t arch_node_attr_is_visible(struct kobject *kobj, struct attribute *attr, int idx) { … } static struct attribute *arch_node_dev_attrs[] = …; const struct attribute_group arch_node_dev_group = …; static void __init arch_update_sysfs_visibility(int nid) { … } #else /* !CONFIG_NUMA */ static void __init arch_update_sysfs_visibility(int nid) {} #endif static bool __init sgx_page_cache_init(void) { … } /* * Update the SGX_LEPUBKEYHASH MSRs to the values specified by caller. * Bare-metal driver requires to update them to hash of enclave's signer * before EINIT. KVM needs to update them to guest's virtual MSR values * before doing EINIT from guest. */ void sgx_update_lepubkeyhash(u64 *lepubkeyhash) { … } const struct file_operations sgx_provision_fops = …; static struct miscdevice sgx_dev_provision = …; /** * sgx_set_attribute() - Update allowed attributes given file descriptor * @allowed_attributes: Pointer to allowed enclave attributes * @attribute_fd: File descriptor for specific attribute * * Append enclave attribute indicated by file descriptor to allowed * attributes. Currently only SGX_ATTR_PROVISIONKEY indicated by * /dev/sgx_provision is supported. * * Return: * -0: SGX_ATTR_PROVISIONKEY is appended to allowed_attributes * -EINVAL: Invalid, or not supported file descriptor */ int sgx_set_attribute(unsigned long *allowed_attributes, unsigned int attribute_fd) { … } EXPORT_SYMBOL_GPL(…); static int __init sgx_init(void) { … } device_initcall(sgx_init);
Codebase Browser
linux