// SPDX-License-Identifier: GPL-2.0-only /* * fs/userfaultfd.c * * Copyright (C) 2007 Davide Libenzi <[email protected]> * Copyright (C) 2008-2009 Red Hat, Inc. * Copyright (C) 2015 Red Hat, Inc. * * Some part derived from fs/eventfd.c (anon inode setup) and * mm/ksm.c (mm hashing). */ #include <linux/list.h> #include <linux/hashtable.h> #include <linux/sched/signal.h> #include <linux/sched/mm.h> #include <linux/mm.h> #include <linux/mm_inline.h> #include <linux/mmu_notifier.h> #include <linux/poll.h> #include <linux/slab.h> #include <linux/seq_file.h> #include <linux/file.h> #include <linux/bug.h> #include <linux/anon_inodes.h> #include <linux/syscalls.h> #include <linux/userfaultfd_k.h> #include <linux/mempolicy.h> #include <linux/ioctl.h> #include <linux/security.h> #include <linux/hugetlb.h> #include <linux/swapops.h> #include <linux/miscdevice.h> #include <linux/uio.h> static int sysctl_unprivileged_userfaultfd __read_mostly; #ifdef CONFIG_SYSCTL static struct ctl_table vm_userfaultfd_table[] = …; #endif static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init; struct userfaultfd_fork_ctx { … }; struct userfaultfd_unmap_ctx { … }; struct userfaultfd_wait_queue { … }; struct userfaultfd_wake_range { … }; /* internal indication that UFFD_API ioctl was successfully executed */ #define UFFD_FEATURE_INITIALIZED … static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) { … } static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) { … } /* * Whether WP_UNPOPULATED is enabled on the uffd context. It is only * meaningful when userfaultfd_wp()==true on the vma and when it's * anonymous. */ bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) { … } static void userfaultfd_set_vm_flags(struct vm_area_struct *vma, vm_flags_t flags) { … } static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, int wake_flags, void *key) { … } /** * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd * context. * @ctx: [in] Pointer to the userfaultfd context. */ static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx) { … } /** * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd * context. * @ctx: [in] Pointer to userfaultfd context. * * The userfaultfd context reference must have been previously acquired either * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget(). */ static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) { … } static inline void msg_init(struct uffd_msg *msg) { … } static inline struct uffd_msg userfault_msg(unsigned long address, unsigned long real_address, unsigned int flags, unsigned long reason, unsigned int features) { … } #ifdef CONFIG_HUGETLB_PAGE /* * Same functionality as userfaultfd_must_wait below with modifications for * hugepmd ranges. */ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, struct vm_fault *vmf, unsigned long reason) { … } #else static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, struct vm_fault *vmf, unsigned long reason) { return false; /* should never get here */ } #endif /* CONFIG_HUGETLB_PAGE */ /* * Verify the pagetables are still not ok after having reigstered into * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any * userfault that has already been resolved, if userfaultfd_read_iter and * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different * threads. */ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, struct vm_fault *vmf, unsigned long reason) { … } static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags) { … } /* * The locking rules involved in returning VM_FAULT_RETRY depending on * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" * recommendation in __lock_page_or_retry is not an understatement. * * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is * not set. * * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not * set, VM_FAULT_RETRY can still be returned if and only if there are * fatal_signal_pending()s, and the mmap_lock must be released before * returning it. */ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) { … } static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, struct userfaultfd_wait_queue *ewq) { … } static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx, struct userfaultfd_wait_queue *ewq) { … } int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) { … } static void dup_fctx(struct userfaultfd_fork_ctx *fctx) { … } void dup_userfaultfd_complete(struct list_head *fcs) { … } void mremap_userfaultfd_prep(struct vm_area_struct *vma, struct vm_userfaultfd_ctx *vm_ctx) { … } void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx, unsigned long from, unsigned long to, unsigned long len) { … } bool userfaultfd_remove(struct vm_area_struct *vma, unsigned long start, unsigned long end) { … } static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, unsigned long start, unsigned long end) { … } int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct list_head *unmaps) { … } void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf) { … } static int userfaultfd_release(struct inode *inode, struct file *file) { … } /* fault_pending_wqh.lock must be hold by the caller */ static inline struct userfaultfd_wait_queue *find_userfault_in( wait_queue_head_t *wqh) { … } static inline struct userfaultfd_wait_queue *find_userfault( struct userfaultfd_ctx *ctx) { … } static inline struct userfaultfd_wait_queue *find_userfault_evt( struct userfaultfd_ctx *ctx) { … } static __poll_t userfaultfd_poll(struct file *file, poll_table *wait) { … } static const struct file_operations userfaultfd_fops; static int resolve_userfault_fork(struct userfaultfd_ctx *new, struct inode *inode, struct uffd_msg *msg) { … } static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, struct uffd_msg *msg, struct inode *inode) { … } static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to) { … } static void __wake_userfault(struct userfaultfd_ctx *ctx, struct userfaultfd_wake_range *range) { … } static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, struct userfaultfd_wake_range *range) { … } static __always_inline int validate_unaligned_range( struct mm_struct *mm, __u64 start, __u64 len) { … } static __always_inline int validate_range(struct mm_struct *mm, __u64 start, __u64 len) { … } static int userfaultfd_register(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, unsigned long arg) { … } /* * userfaultfd_wake may be used in combination with the * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches. */ static int userfaultfd_wake(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static int userfaultfd_copy(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg) { … } bool userfaultfd_wp_async(struct vm_area_struct *vma) { … } static inline unsigned int uffd_ctx_features(__u64 user_features) { … } static int userfaultfd_move(struct userfaultfd_ctx *ctx, unsigned long arg) { … } /* * userland asks for a certain API version and we return which bits * and ioctl commands are implemented in this kernel for such API * version or -EINVAL if unknown. */ static int userfaultfd_api(struct userfaultfd_ctx *ctx, unsigned long arg) { … } static long userfaultfd_ioctl(struct file *file, unsigned cmd, unsigned long arg) { … } #ifdef CONFIG_PROC_FS static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) { … } #endif static const struct file_operations userfaultfd_fops = …; static void init_once_userfaultfd_ctx(void *mem) { … } static int new_userfaultfd(int flags) { … } static inline bool userfaultfd_syscall_allowed(int flags) { … } SYSCALL_DEFINE1(userfaultfd, int, flags) { … } static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags) { … } static const struct file_operations userfaultfd_dev_fops = …; static struct miscdevice userfaultfd_misc = …; static int __init userfaultfd_init(void) { … } __initcall(userfaultfd_init);
Codebase Browser
linux