// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2015 - 2018 Intel Corporation. */ #include <linux/io.h> #include <rdma/rdma_vt.h> #include <rdma/rdmavt_qp.h> #include "hfi.h" #include "qp.h" #include "rc.h" #include "verbs_txreq.h" #include "trace.h" struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev, u8 *prev_ack, bool *scheduled) __must_hold(&qp->s_lock) { … } /** * make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) * @dev: the device for this QP * @qp: a pointer to the QP * @ohdr: a pointer to the IB header being constructed * @ps: the xmit packet state * * Return 1 if constructed; otherwise, return 0. * Note that we are in the responder's side of the QP context. * Note the QP s_lock must be held. */ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, struct ib_other_headers *ohdr, struct hfi1_pkt_state *ps) { … } /** * hfi1_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) * @qp: a pointer to the QP * @ps: the current packet state * * Assumes s_lock is held. * * Return 1 if constructed; otherwise, return 0. */ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) { … } static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, struct ib_other_headers *ohdr, u32 bth0, u32 bth1) { … } static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn) { … } static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, u32 *nwords) { … } static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet, struct hfi1_opa_header *opa_hdr, u8 sc5, bool is_fecn, u64 *pbc_flags, u32 *hwords, u32 *nwords) { … } hfi1_make_rc_ack; /* We support only two types - 9B and 16B for now */ static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = …; /* * hfi1_send_rc_ack - Construct an ACK packet and send it * * This is called from hfi1_rc_rcv() and handle_receive_interrupt(). * Note that RDMA reads and atomics are handled in the * send side QP state and send engine. */ void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn) { … } /** * update_num_rd_atomic - update the qp->s_num_rd_atomic * @qp: the QP * @psn: the packet sequence number to restart at * @wqe: the wqe * * This is called from reset_psn() to update qp->s_num_rd_atomic * for the current wqe. * Called at interrupt level with the QP s_lock held. */ static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn, struct rvt_swqe *wqe) { … } /** * reset_psn - reset the QP state to send starting from PSN * @qp: the QP * @psn: the packet sequence number to restart at * * This is called from hfi1_rc_rcv() to process an incoming RC ACK * for the given QP. * Called at interrupt level with the QP s_lock held. */ static void reset_psn(struct rvt_qp *qp, u32 psn) { … } /* * Back up requester to resend the last un-ACKed request. * The QP r_lock and s_lock should be held and interrupts disabled. */ void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait) { … } /* * Set qp->s_sending_psn to the next PSN after the given one. * This would be psn+1 except when RDMA reads or TID RDMA ops * are present. */ static void reset_sending_psn(struct rvt_qp *qp, u32 psn) { … } /** * hfi1_rc_verbs_aborted - handle abort status * @qp: the QP * @opah: the opa header * * This code modifies both ACK bit in BTH[2] * and the s_flags to go into send one mode. * * This serves to throttle the send engine to only * send a single packet in the likely case the * a link has gone down. */ void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah) { … } /* * This should be called with the QP s_lock held and interrupts disabled. */ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) { … } static inline void update_last_psn(struct rvt_qp *qp, u32 psn) { … } /* * Generate a SWQE completion. * This is similar to hfi1_send_complete but has to check to be sure * that the SGEs are not being referenced if the SWQE is being resent. */ struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, struct rvt_swqe *wqe, struct hfi1_ibport *ibp) { … } static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd) { … } /** * update_qp_retry_state - Update qp retry state. * @qp: the QP * @psn: the packet sequence number of the TID RDMA WRITE RESP. * @spsn: The start psn for the given TID RDMA WRITE swqe. * @lpsn: The last psn for the given TID RDMA WRITE swqe. * * This function is called to update the qp retry state upon * receiving a TID WRITE RESP after the qp is scheduled to retry * a request. */ static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn, u32 lpsn) { … } /* * do_rc_ack - process an incoming RC ACK * @qp: the QP the ACK came in on * @psn: the packet sequence number of the ACK * @opcode: the opcode of the request that resulted in the ACK * * This is called from rc_rcv_resp() to process an incoming RC ACK * for the given QP. * May be called at interrupt level, with the QP s_lock held. * Returns 1 if OK, 0 if current operation should be aborted (NAK). */ int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, u64 val, struct hfi1_ctxtdata *rcd) { … } /* * We have seen an out of sequence RDMA read middle or last packet. * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE. */ static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn, struct hfi1_ctxtdata *rcd) { … } /** * rc_rcv_resp - process an incoming RC response packet * @packet: data packet information * * This is called from hfi1_rc_rcv() to process an incoming RC response * packet for the given QP. * Called at interrupt level. */ static void rc_rcv_resp(struct hfi1_packet *packet) { … } static inline void rc_cancel_ack(struct rvt_qp *qp) { … } /** * rc_rcv_error - process an incoming duplicate or error RC packet * @ohdr: the other headers for this packet * @data: the packet data * @qp: the QP for this packet * @opcode: the opcode for this packet * @psn: the packet sequence number for this packet * @diff: the difference between the PSN and the expected PSN * @rcd: the receive context * * This is called from hfi1_rc_rcv() to process an unexpected * incoming RC packet for the given QP. * Called at interrupt level. * Return 1 if no more processing is needed; otherwise return 0 to * schedule a response to be sent. */ static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data, struct rvt_qp *qp, u32 opcode, u32 psn, int diff, struct hfi1_ctxtdata *rcd) { … } static void log_cca_event(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn, u32 rqpn, u8 svc_type) { … } void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn, u32 rqpn, u8 svc_type) { … } /** * hfi1_rc_rcv - process an incoming RC packet * @packet: data packet information * * This is called from qp_rcv() to process an incoming RC packet * for the given QP. * May be called at interrupt level. */ void hfi1_rc_rcv(struct hfi1_packet *packet) { … } void hfi1_rc_hdrerr( struct hfi1_ctxtdata *rcd, struct hfi1_packet *packet, struct rvt_qp *qp) { … }
Codebase Browser
linux