// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2020 Cornelis Networks, Inc. * Copyright(c) 2015-2018 Intel Corporation. */ #include <asm/page.h> #include <linux/string.h> #include "mmu_rb.h" #include "user_exp_rcv.h" #include "trace.h" static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt, struct exp_tid_set *set, struct hfi1_filedata *fd); static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages); static int set_rcvarray_entry(struct hfi1_filedata *fd, struct tid_user_buf *tbuf, u32 rcventry, struct tid_group *grp, u16 pageidx, unsigned int npages); static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata, struct tid_rb_node *tnode); static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq); static bool tid_cover_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq); static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *, struct tid_group *grp, u16 count, u32 *tidlist, unsigned int *tididx, unsigned int *pmapped); static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo); static void __clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node); static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node); static const struct mmu_interval_notifier_ops tid_mn_ops = …; static const struct mmu_interval_notifier_ops tid_cover_ops = …; /* * Initialize context and file private data needed for Expected * receive caching. This needs to be done after the context has * been configured with the eager/expected RcvEntry counts. */ int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd, struct hfi1_ctxtdata *uctxt) { … } void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd) { … } /* * Release pinned receive buffer pages. * * @mapped: true if the pages have been DMA mapped. false otherwise. * @idx: Index of the first page to unpin. * @npages: No of pages to unpin. * * If the pages have been DMA mapped (indicated by mapped parameter), their * info will be passed via a struct tid_rb_node. If they haven't been mapped, * their info will be passed via a struct tid_user_buf. */ static void unpin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf, struct tid_rb_node *node, unsigned int idx, unsigned int npages, bool mapped) { … } /* * Pin receive buffer pages. */ static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf) { … } /* * RcvArray entry allocation for Expected Receives is done by the * following algorithm: * * The context keeps 3 lists of groups of RcvArray entries: * 1. List of empty groups - tid_group_list * This list is created during user context creation and * contains elements which describe sets (of 8) of empty * RcvArray entries. * 2. List of partially used groups - tid_used_list * This list contains sets of RcvArray entries which are * not completely used up. Another mapping request could * use some of all of the remaining entries. * 3. List of full groups - tid_full_list * This is the list where sets that are completely used * up go. * * An attempt to optimize the usage of RcvArray entries is * made by finding all sets of physically contiguous pages in a * user's buffer. * These physically contiguous sets are further split into * sizes supported by the receive engine of the HFI. The * resulting sets of pages are stored in struct tid_pageset, * which describes the sets as: * * .count - number of pages in this set * * .idx - starting index into struct page ** array * of this set * * From this point on, the algorithm deals with the page sets * described above. The number of pagesets is divided by the * RcvArray group size to produce the number of full groups * needed. * * Groups from the 3 lists are manipulated using the following * rules: * 1. For each set of 8 pagesets, a complete group from * tid_group_list is taken, programmed, and moved to * the tid_full_list list. * 2. For all remaining pagesets: * 2.1 If the tid_used_list is empty and the tid_group_list * is empty, stop processing pageset and return only * what has been programmed up to this point. * 2.2 If the tid_used_list is empty and the tid_group_list * is not empty, move a group from tid_group_list to * tid_used_list. * 2.3 For each group is tid_used_group, program as much as * can fit into the group. If the group becomes fully * used, move it to tid_full_list. */ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd, struct hfi1_tid_info *tinfo) { … } int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd, struct hfi1_tid_info *tinfo) { … } int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd, struct hfi1_tid_info *tinfo) { … } static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages) { … } /** * program_rcvarray() - program an RcvArray group with receive buffers * @fd: filedata pointer * @tbuf: pointer to struct tid_user_buf that has the user buffer starting * virtual address, buffer length, page pointers, pagesets (array of * struct tid_pageset holding information on physically contiguous * chunks from the user buffer), and other fields. * @grp: RcvArray group * @count: number of struct tid_pageset's to program * @tidlist: the array of u32 elements when the information about the * programmed RcvArray entries is to be encoded. * @tididx: starting offset into tidlist * @pmapped: (output parameter) number of pages programmed into the RcvArray * entries. * * This function will program up to 'count' number of RcvArray entries from the * group 'grp'. To make best use of write-combining writes, the function will * perform writes to the unused RcvArray entries which will be ignored by the * HW. Each RcvArray entry will be programmed with a physically contiguous * buffer chunk from the user's virtual buffer. * * Return: * -EINVAL if the requested count is larger than the size of the group, * -ENOMEM or -EFAULT on error from set_rcvarray_entry(), or * number of RcvArray entries programmed. */ static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf, struct tid_group *grp, u16 count, u32 *tidlist, unsigned int *tididx, unsigned int *pmapped) { … } static int set_rcvarray_entry(struct hfi1_filedata *fd, struct tid_user_buf *tbuf, u32 rcventry, struct tid_group *grp, u16 pageidx, unsigned int npages) { … } static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo) { … } static void __clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node) { … } static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node) { … } /* * As a simple helper for hfi1_user_exp_rcv_free, this function deals with * clearing nodes in the non-cached case. */ static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt, struct exp_tid_set *set, struct hfi1_filedata *fd) { … } static bool tid_rb_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { … } static bool tid_cover_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { … } static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata, struct tid_rb_node *tnode) { … }
Codebase Browser
linux