// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2020 Cornelis Networks, Inc. * Copyright(c) 2015-2020 Intel Corporation. */ #include <linux/poll.h> #include <linux/cdev.h> #include <linux/vmalloc.h> #include <linux/io.h> #include <linux/sched/mm.h> #include <linux/bitmap.h> #include <rdma/ib.h> #include "hfi.h" #include "pio.h" #include "device.h" #include "common.h" #include "trace.h" #include "mmu_rb.h" #include "user_sdma.h" #include "user_exp_rcv.h" #include "aspm.h" #undef pr_fmt #define pr_fmt(fmt) … #define SEND_CTXT_HALT_TIMEOUT … /* * File operation functions */ static int hfi1_file_open(struct inode *inode, struct file *fp); static int hfi1_file_close(struct inode *inode, struct file *fp); static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from); static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt); static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma); static u64 kvirt_to_phys(void *addr); static int assign_ctxt(struct hfi1_filedata *fd, unsigned long arg, u32 len); static void init_subctxts(struct hfi1_ctxtdata *uctxt, const struct hfi1_user_info *uinfo); static int init_user_ctxt(struct hfi1_filedata *fd, struct hfi1_ctxtdata *uctxt); static void user_init(struct hfi1_ctxtdata *uctxt); static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len); static int get_base_info(struct hfi1_filedata *fd, unsigned long arg, u32 len); static int user_exp_rcv_setup(struct hfi1_filedata *fd, unsigned long arg, u32 len); static int user_exp_rcv_clear(struct hfi1_filedata *fd, unsigned long arg, u32 len); static int user_exp_rcv_invalid(struct hfi1_filedata *fd, unsigned long arg, u32 len); static int setup_base_ctxt(struct hfi1_filedata *fd, struct hfi1_ctxtdata *uctxt); static int setup_subctxt(struct hfi1_ctxtdata *uctxt); static int find_sub_ctxt(struct hfi1_filedata *fd, const struct hfi1_user_info *uinfo); static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd, struct hfi1_user_info *uinfo, struct hfi1_ctxtdata **cd); static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt); static __poll_t poll_urgent(struct file *fp, struct poll_table_struct *pt); static __poll_t poll_next(struct file *fp, struct poll_table_struct *pt); static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt, unsigned long arg); static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg); static int ctxt_reset(struct hfi1_ctxtdata *uctxt); static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt, unsigned long arg); static vm_fault_t vma_fault(struct vm_fault *vmf); static long hfi1_file_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); static const struct file_operations hfi1_file_ops = …; static const struct vm_operations_struct vm_ops = …; /* * Types of memories mapped into user processes' space */ enum mmap_types { … }; /* * Masks and offsets defining the mmap tokens */ #define HFI1_MMAP_OFFSET_MASK … #define HFI1_MMAP_OFFSET_SHIFT … #define HFI1_MMAP_SUBCTXT_MASK … #define HFI1_MMAP_SUBCTXT_SHIFT … #define HFI1_MMAP_CTXT_MASK … #define HFI1_MMAP_CTXT_SHIFT … #define HFI1_MMAP_TYPE_MASK … #define HFI1_MMAP_TYPE_SHIFT … #define HFI1_MMAP_MAGIC_MASK … #define HFI1_MMAP_MAGIC_SHIFT … #define HFI1_MMAP_MAGIC … #define HFI1_MMAP_TOKEN_SET(field, val) … #define HFI1_MMAP_TOKEN_GET(field, token) … #define HFI1_MMAP_TOKEN(type, ctxt, subctxt, addr) … #define dbg(fmt, ...) … static inline int is_valid_mmap(u64 token) { … } static int hfi1_file_open(struct inode *inode, struct file *fp) { … } static long hfi1_file_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) { … } static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from) { … } static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf, u64 memaddr, void *memvirt, dma_addr_t memdma, ssize_t memlen, struct vm_area_struct *vma) { … } static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) { … } /* * Local (non-chip) user memory is not mapped right away but as it is * accessed by the user-level code. */ static vm_fault_t vma_fault(struct vm_fault *vmf) { … } static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt) { … } static int hfi1_file_close(struct inode *inode, struct file *fp) { … } /* * Convert kernel *virtual* addresses to physical addresses. * This is used to vmalloc'ed addresses. */ static u64 kvirt_to_phys(void *addr) { … } /** * complete_subctxt - complete sub-context info * @fd: valid filedata pointer * * Sub-context info can only be set up after the base context * has been completed. This is indicated by the clearing of the * HFI1_CTXT_BASE_UINIT bit. * * Wait for the bit to be cleared, and then complete the subcontext * initialization. * */ static int complete_subctxt(struct hfi1_filedata *fd) { … } static int assign_ctxt(struct hfi1_filedata *fd, unsigned long arg, u32 len) { … } /** * match_ctxt - match context * @fd: valid filedata pointer * @uinfo: user info to compare base context with * @uctxt: context to compare uinfo to. * * Compare the given context with the given information to see if it * can be used for a sub context. */ static int match_ctxt(struct hfi1_filedata *fd, const struct hfi1_user_info *uinfo, struct hfi1_ctxtdata *uctxt) { … } /** * find_sub_ctxt - fund sub-context * @fd: valid filedata pointer * @uinfo: matching info to use to find a possible context to share. * * The hfi1_mutex must be held when this function is called. It is * necessary to ensure serialized creation of shared contexts. * * Return: * 0 No sub-context found * 1 Subcontext found and allocated * errno EINVAL (incorrect parameters) * EBUSY (all sub contexts in use) */ static int find_sub_ctxt(struct hfi1_filedata *fd, const struct hfi1_user_info *uinfo) { … } static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd, struct hfi1_user_info *uinfo, struct hfi1_ctxtdata **rcd) { … } static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt) { … } static void init_subctxts(struct hfi1_ctxtdata *uctxt, const struct hfi1_user_info *uinfo) { … } static int setup_subctxt(struct hfi1_ctxtdata *uctxt) { … } static void user_init(struct hfi1_ctxtdata *uctxt) { … } static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len) { … } static int init_user_ctxt(struct hfi1_filedata *fd, struct hfi1_ctxtdata *uctxt) { … } static int setup_base_ctxt(struct hfi1_filedata *fd, struct hfi1_ctxtdata *uctxt) { … } static int get_base_info(struct hfi1_filedata *fd, unsigned long arg, u32 len) { … } /** * user_exp_rcv_setup - Set up the given tid rcv list * @fd: file data of the current driver instance * @arg: ioctl argumnent for user space information * @len: length of data structure associated with ioctl command * * Wrapper to validate ioctl information before doing _rcv_setup. * */ static int user_exp_rcv_setup(struct hfi1_filedata *fd, unsigned long arg, u32 len) { … } /** * user_exp_rcv_clear - Clear the given tid rcv list * @fd: file data of the current driver instance * @arg: ioctl argumnent for user space information * @len: length of data structure associated with ioctl command * * The hfi1_user_exp_rcv_clear() can be called from the error path. Because * of this, we need to use this wrapper to copy the user space information * before doing the clear. */ static int user_exp_rcv_clear(struct hfi1_filedata *fd, unsigned long arg, u32 len) { … } /** * user_exp_rcv_invalid - Invalidate the given tid rcv list * @fd: file data of the current driver instance * @arg: ioctl argumnent for user space information * @len: length of data structure associated with ioctl command * * Wrapper to validate ioctl information before doing _rcv_invalid. * */ static int user_exp_rcv_invalid(struct hfi1_filedata *fd, unsigned long arg, u32 len) { … } static __poll_t poll_urgent(struct file *fp, struct poll_table_struct *pt) { … } static __poll_t poll_next(struct file *fp, struct poll_table_struct *pt) { … } /* * Find all user contexts in use, and set the specified bit in their * event mask. * See also find_ctxt() for a similar use, that is specific to send buffers. */ int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit) { … } /** * manage_rcvq - manage a context's receive queue * @uctxt: the context * @subctxt: the sub-context * @arg: start/stop action to carry out * * start_stop == 0 disables receive on the context, for use in queue * overflow conditions. start_stop==1 re-enables, to be used to * re-init the software copy of the head register */ static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt, unsigned long arg) { … } /* * clear the event notifier events for this context. * User process then performs actions appropriate to bit having been * set, if desired, and checks again in future. */ static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt, unsigned long arg) { … } static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg) { … } /** * ctxt_reset - Reset the user context * @uctxt: valid user context */ static int ctxt_reset(struct hfi1_ctxtdata *uctxt) { … } static void user_remove(struct hfi1_devdata *dd) { … } static int user_add(struct hfi1_devdata *dd) { … } /* * Create per-unit files in /dev */ int hfi1_device_create(struct hfi1_devdata *dd) { … } /* * Remove per-unit files in /dev * void, core kernel returns no errors for this stuff */ void hfi1_device_remove(struct hfi1_devdata *dd) { … }
Codebase Browser
linux