// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2016 - 2020 Intel Corporation. */ #include <linux/hash.h> #include <linux/bitops.h> #include <linux/lockdep.h> #include <linux/vmalloc.h> #include <linux/slab.h> #include <rdma/ib_verbs.h> #include <rdma/ib_hdrs.h> #include <rdma/opa_addr.h> #include <rdma/uverbs_ioctl.h> #include "qp.h" #include "vt.h" #include "trace.h" #define RVT_RWQ_COUNT_THRESHOLD … static void rvt_rc_timeout(struct timer_list *t); static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, enum ib_qp_type type); /* * Convert the AETH RNR timeout code into the number of microseconds. */ static const u32 ib_rvt_rnr_table[32] = …; /* * Note that it is OK to post send work requests in the SQE and ERR * states; rvt_do_send() will process them and generate error * completions as per IB 1.2 C10-96. */ const int ib_rvt_state_ops[IB_QPS_ERR + 1] = …; EXPORT_SYMBOL(…); /* platform specific: return the last level cache (llc) size, in KiB */ static int rvt_wss_llc_size(void) { … } /* platform specific: cacheless copy */ static void cacheless_memcpy(void *dst, void *src, size_t n) { … } void rvt_wss_exit(struct rvt_dev_info *rdi) { … } /* * rvt_wss_init - Init wss data structures * * Return: 0 on success */ int rvt_wss_init(struct rvt_dev_info *rdi) { … } /* * Advance the clean counter. When the clean period has expired, * clean an entry. * * This is implemented in atomics to avoid locking. Because multiple * variables are involved, it can be racy which can lead to slightly * inaccurate information. Since this is only a heuristic, this is * OK. Any innaccuracies will clean themselves out as the counter * advances. That said, it is unlikely the entry clean operation will * race - the next possible racer will not start until the next clean * period. * * The clean counter is implemented as a decrement to zero. When zero * is reached an entry is cleaned. */ static void wss_advance_clean_counter(struct rvt_wss *wss) { … } /* * Insert the given address into the working set array. */ static void wss_insert(struct rvt_wss *wss, void *address) { … } /* * Is the working set larger than the threshold? */ static inline bool wss_exceeds_threshold(struct rvt_wss *wss) { … } static void get_map_page(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map) { … } /** * init_qpn_table - initialize the QP number table for a device * @rdi: rvt dev struct * @qpt: the QPN table */ static int init_qpn_table(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt) { … } /** * free_qpn_table - free the QP number table for a device * @qpt: the QPN table */ static void free_qpn_table(struct rvt_qpn_table *qpt) { … } /** * rvt_driver_qp_init - Init driver qp resources * @rdi: rvt dev strucutre * * Return: 0 on success */ int rvt_driver_qp_init(struct rvt_dev_info *rdi) { … } /** * rvt_free_qp_cb - callback function to reset a qp * @qp: the qp to reset * @v: a 64-bit value * * This function resets the qp and removes it from the * qp hash table. */ static void rvt_free_qp_cb(struct rvt_qp *qp, u64 v) { … } /** * rvt_free_all_qps - check for QPs still in use * @rdi: rvt device info structure * * There should not be any QPs still in use. * Free memory for table. * Return the number of QPs still in use. */ static unsigned rvt_free_all_qps(struct rvt_dev_info *rdi) { … } /** * rvt_qp_exit - clean up qps on device exit * @rdi: rvt dev structure * * Check for qp leaks and free resources. */ void rvt_qp_exit(struct rvt_dev_info *rdi) { … } static inline unsigned mk_qpn(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map, unsigned off) { … } /** * alloc_qpn - Allocate the next available qpn or zero/one for QP type * IB_QPT_SMI/IB_QPT_GSI * @rdi: rvt device info structure * @qpt: queue pair number table pointer * @type: the QP type * @port_num: IB port number, 1 based, comes from core * @exclude_prefix: prefix of special queue pair number being allocated * * Return: The queue pair number */ static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt, enum ib_qp_type type, u8 port_num, u8 exclude_prefix) { … } /** * rvt_clear_mr_refs - Drop help mr refs * @qp: rvt qp data structure * @clr_sends: If shoudl clear send side or not */ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends) { … } /** * rvt_swqe_has_lkey - return true if lkey is used by swqe * @wqe: the send wqe * @lkey: the lkey * * Test the swqe for using lkey */ static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey) { … } /** * rvt_qp_sends_has_lkey - return true is qp sends use lkey * @qp: the rvt_qp * @lkey: the lkey */ static bool rvt_qp_sends_has_lkey(struct rvt_qp *qp, u32 lkey) { … } /** * rvt_qp_acks_has_lkey - return true if acks have lkey * @qp: the qp * @lkey: the lkey */ static bool rvt_qp_acks_has_lkey(struct rvt_qp *qp, u32 lkey) { … } /** * rvt_qp_mr_clean - clean up remote ops for lkey * @qp: the qp * @lkey: the lkey that is being de-registered * * This routine checks if the lkey is being used by * the qp. * * If so, the qp is put into an error state to elminate * any references from the qp. */ void rvt_qp_mr_clean(struct rvt_qp *qp, u32 lkey) { … } /** * rvt_remove_qp - remove qp form table * @rdi: rvt dev struct * @qp: qp to remove * * Remove the QP from the table so it can't be found asynchronously by * the receive routine. */ static void rvt_remove_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) { … } /** * rvt_alloc_rq - allocate memory for user or kernel buffer * @rq: receive queue data structure * @size: number of request queue entries * @node: The NUMA node * @udata: True if user data is available or not false * * Return: If memory allocation failed, return -ENONEM * This function is used by both shared receive * queues and non-shared receive queues to allocate * memory. */ int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node, struct ib_udata *udata) { … } /** * rvt_init_qp - initialize the QP state to the reset state * @rdi: rvt dev struct * @qp: the QP to init or reinit * @type: the QP type * * This function is called from both rvt_create_qp() and * rvt_reset_qp(). The difference is that the reset * patch the necessary locks to protect against concurent * access. */ static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, enum ib_qp_type type) { … } /** * _rvt_reset_qp - initialize the QP state to the reset state * @rdi: rvt dev struct * @qp: the QP to reset * @type: the QP type * * r_lock, s_hlock, and s_lock are required to be held by the caller */ static void _rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, enum ib_qp_type type) __must_hold(&qp->s_lock) __must_hold(&qp->s_hlock) __must_hold(&qp->r_lock) { … } /** * rvt_reset_qp - initialize the QP state to the reset state * @rdi: the device info * @qp: the QP to reset * @type: the QP type * * This is the wrapper function to acquire the r_lock, s_hlock, and s_lock * before calling _rvt_reset_qp(). */ static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, enum ib_qp_type type) { … } /** * rvt_free_qpn - Free a qpn from the bit map * @qpt: QP table * @qpn: queue pair number to free */ static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn) { … } /** * get_allowed_ops - Given a QP type return the appropriate allowed OP * @type: valid, supported, QP type */ static u8 get_allowed_ops(enum ib_qp_type type) { … } /** * free_ud_wq_attr - Clean up AH attribute cache for UD QPs * @qp: Valid QP with allowed_ops set * * The rvt_swqe data structure being used is a union, so this is * only valid for UD QPs. */ static void free_ud_wq_attr(struct rvt_qp *qp) { … } /** * alloc_ud_wq_attr - AH attribute cache for UD QPs * @qp: Valid QP with allowed_ops set * @node: Numa node for allocation * * The rvt_swqe data structure being used is a union, so this is * only valid for UD QPs. */ static int alloc_ud_wq_attr(struct rvt_qp *qp, int node) { … } /** * rvt_create_qp - create a queue pair for a device * @ibqp: the queue pair * @init_attr: the attributes of the queue pair * @udata: user data for libibverbs.so * * Queue pair creation is mostly an rvt issue. However, drivers have their own * unique idea of what queue pair numbers mean. For instance there is a reserved * range for PSM. * * Return: 0 on success, otherwise returns an errno. * * Called by the ib_create_qp() core verbs function. */ int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { … } /** * rvt_error_qp - put a QP into the error state * @qp: the QP to put into the error state * @err: the receive completion error to signal if a RWQE is active * * Flushes both send and receive work queues. * * Return: true if last WQE event should be generated. * The QP r_lock and s_lock should be held and interrupts disabled. * If we are already in error state, just return. */ int rvt_error_qp(struct rvt_qp *qp, enum ib_wc_status err) { … } EXPORT_SYMBOL(…); /* * Put the QP into the hash table. * The hash table holds a reference to the QP. */ static void rvt_insert_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp) { … } /** * rvt_modify_qp - modify the attributes of a queue pair * @ibqp: the queue pair who's attributes we're modifying * @attr: the new attributes * @attr_mask: the mask of attributes to modify * @udata: user data for libibverbs.so * * Return: 0 on success, otherwise returns an errno. */ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { … } /** * rvt_destroy_qp - destroy a queue pair * @ibqp: the queue pair to destroy * @udata: unused by the driver * * Note that this can be called while the QP is actively sending or * receiving! * * Return: 0 on success. */ int rvt_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { … } /** * rvt_query_qp - query an ipbq * @ibqp: IB qp to query * @attr: attr struct to fill in * @attr_mask: attr mask ignored * @init_attr: struct to fill in * * Return: always 0 */ int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_qp_init_attr *init_attr) { … } /** * rvt_post_recv - post a receive on a QP * @ibqp: the QP to post the receive on * @wr: the WR to post * @bad_wr: the first bad WR is put here * * This may be called from interrupt context. * * Return: 0 on success otherwise errno */ int rvt_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { … } /** * rvt_qp_valid_operation - validate post send wr request * @qp: the qp * @post_parms: the post send table for the driver * @wr: the work request * * The routine validates the operation based on the * validation table an returns the length of the operation * which can extend beyond the ib_send_bw. Operation * dependent flags key atomic operation validation. * * There is an exception for UD qps that validates the pd and * overrides the length to include the additional UD specific * length. * * Returns a negative error or the length of the work request * for building the swqe. */ static inline int rvt_qp_valid_operation( struct rvt_qp *qp, const struct rvt_operation_params *post_parms, const struct ib_send_wr *wr) { … } /** * rvt_qp_is_avail - determine queue capacity * @qp: the qp * @rdi: the rdmavt device * @reserved_op: is reserved operation * * This assumes the s_hlock is held but the s_last * qp variable is uncontrolled. * * For non reserved operations, the qp->s_avail * may be changed. * * The return value is zero or a -ENOMEM. */ static inline int rvt_qp_is_avail( struct rvt_qp *qp, struct rvt_dev_info *rdi, bool reserved_op) { … } /** * rvt_post_one_wr - post one RC, UC, or UD send work request * @qp: the QP to post on * @wr: the work request to send * @call_send: kick the send engine into gear */ static int rvt_post_one_wr(struct rvt_qp *qp, const struct ib_send_wr *wr, bool *call_send) { … } /** * rvt_post_send - post a send on a QP * @ibqp: the QP to post the send on * @wr: the list of work requests to post * @bad_wr: the first bad WR is put here * * This may be called from interrupt context. * * Return: 0 on success else errno */ int rvt_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { … } /** * rvt_post_srq_recv - post a receive on a shared receive queue * @ibsrq: the SRQ to post the receive on * @wr: the list of work requests to post * @bad_wr: A pointer to the first WR to cause a problem is put here * * This may be called from interrupt context. * * Return: 0 on success else errno */ int rvt_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { … } /* * rvt used the internal kernel struct as part of its ABI, for now make sure * the kernel struct does not change layout. FIXME: rvt should never cast the * user struct to a kernel struct. */ static struct ib_sge *rvt_cast_sge(struct rvt_wqe_sge *sge) { … } /* * Validate a RWQE and fill in the SGE state. * Return 1 if OK. */ static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) { … } /** * get_rvt_head - get head indices of the circular buffer * @rq: data structure for request queue entry * @ip: the QP * * Return - head index value */ static inline u32 get_rvt_head(struct rvt_rq *rq, void *ip) { … } /** * rvt_get_rwqe - copy the next RWQE into the QP's RWQE * @qp: the QP * @wr_id_only: update qp->r_wr_id only, not qp->r_sge * * Return -1 if there is a local error, 0 if no RWQE is available, * otherwise return 1. * * Can be called from interrupt level. */ int rvt_get_rwqe(struct rvt_qp *qp, bool wr_id_only) { … } EXPORT_SYMBOL(…); /** * rvt_comm_est - handle trap with QP established * @qp: the QP */ void rvt_comm_est(struct rvt_qp *qp) { … } EXPORT_SYMBOL(…); void rvt_rc_error(struct rvt_qp *qp, enum ib_wc_status err) { … } EXPORT_SYMBOL(…); /* * rvt_rnr_tbl_to_usec - return index into ib_rvt_rnr_table * @index - the index * return usec from an index into ib_rvt_rnr_table */ unsigned long rvt_rnr_tbl_to_usec(u32 index) { … } EXPORT_SYMBOL(…); static inline unsigned long rvt_aeth_to_usec(u32 aeth) { … } /* * rvt_add_retry_timer_ext - add/start a retry timer * @qp - the QP * @shift - timeout shift to wait for multiple packets * add a retry timer on the QP */ void rvt_add_retry_timer_ext(struct rvt_qp *qp, u8 shift) { … } EXPORT_SYMBOL(…); /** * rvt_add_rnr_timer - add/start an rnr timer on the QP * @qp: the QP * @aeth: aeth of RNR timeout, simulated aeth for loopback */ void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth) { … } EXPORT_SYMBOL(…); /** * rvt_stop_rc_timers - stop all timers * @qp: the QP * stop any pending timers */ void rvt_stop_rc_timers(struct rvt_qp *qp) { … } EXPORT_SYMBOL(…); /** * rvt_stop_rnr_timer - stop an rnr timer * @qp: the QP * * stop an rnr timer and return if the timer * had been pending. */ static void rvt_stop_rnr_timer(struct rvt_qp *qp) { … } /** * rvt_del_timers_sync - wait for any timeout routines to exit * @qp: the QP */ void rvt_del_timers_sync(struct rvt_qp *qp) { … } EXPORT_SYMBOL(…); /* * This is called from s_timer for missing responses. */ static void rvt_rc_timeout(struct timer_list *t) { … } /* * This is called from s_timer for RNR timeouts. */ enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t) { … } EXPORT_SYMBOL(…); /** * rvt_qp_iter_init - initial for QP iteration * @rdi: rvt devinfo * @v: u64 value * @cb: user-defined callback * * This returns an iterator suitable for iterating QPs * in the system. * * The @cb is a user-defined callback and @v is a 64-bit * value passed to and relevant for processing in the * @cb. An example use case would be to alter QP processing * based on criteria not part of the rvt_qp. * * Use cases that require memory allocation to succeed * must preallocate appropriately. * * Return: a pointer to an rvt_qp_iter or NULL */ struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi, u64 v, void (*cb)(struct rvt_qp *qp, u64 v)) { … } EXPORT_SYMBOL(…); /** * rvt_qp_iter_next - return the next QP in iter * @iter: the iterator * * Fine grained QP iterator suitable for use * with debugfs seq_file mechanisms. * * Updates iter->qp with the current QP when the return * value is 0. * * Return: 0 - iter->qp is valid 1 - no more QPs */ int rvt_qp_iter_next(struct rvt_qp_iter *iter) __must_hold(RCU) { … } EXPORT_SYMBOL(…); /** * rvt_qp_iter - iterate all QPs * @rdi: rvt devinfo * @v: a 64-bit value * @cb: a callback * * This provides a way for iterating all QPs. * * The @cb is a user-defined callback and @v is a 64-bit * value passed to and relevant for processing in the * cb. An example use case would be to alter QP processing * based on criteria not part of the rvt_qp. * * The code has an internal iterator to simplify * non seq_file use cases. */ void rvt_qp_iter(struct rvt_dev_info *rdi, u64 v, void (*cb)(struct rvt_qp *qp, u64 v)) { … } EXPORT_SYMBOL(…); /* * This should be called with s_lock and r_lock held. */ void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, enum ib_wc_status status) { … } EXPORT_SYMBOL(…); /** * rvt_copy_sge - copy data to SGE memory * @qp: associated QP * @ss: the SGE state * @data: the data to copy * @length: the length of the data * @release: boolean to release MR * @copy_last: do a separate copy of the last 8 bytes */ void rvt_copy_sge(struct rvt_qp *qp, struct rvt_sge_state *ss, void *data, u32 length, bool release, bool copy_last) { … } EXPORT_SYMBOL(…); static enum ib_wc_status loopback_qp_drop(struct rvt_ibport *rvp, struct rvt_qp *sqp) { … } /** * rvt_ruc_loopback - handle UC and RC loopback requests * @sqp: the sending QP * * This is called from rvt_do_send() to forward a WQE addressed to the same HFI * Note that although we are single threaded due to the send engine, we still * have to protect against post_send(). We don't have to worry about * receive interrupts since this is a connected protocol and all packets * will pass through here. */ void rvt_ruc_loopback(struct rvt_qp *sqp) { … } EXPORT_SYMBOL(…);
Codebase Browser
linux