linux/drivers/infiniband/hw/mlx5/wr.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
 */

#include <linux/gfp.h>
#include <linux/mlx5/qp.h>
#include <linux/mlx5/driver.h>
#include "wr.h"
#include "umr.h"

static const u32 mlx5_ib_opcode[] = …;

int mlx5r_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
{ … }

static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
					  u64 remote_addr, u32 rkey)
{ … }

static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
			void **seg, int *size, void **cur_edge)
{ … }

static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
			     const struct ib_send_wr *wr)
{ … }

static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
{ … }

static __be64 frwr_mkey_mask(bool atomic)
{ … }

static __be64 sig_mkey_mask(void)
{ … }

static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
			    struct mlx5_ib_mr *mr, u8 flags, bool atomic)
{ … }

static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
{ … }

static u8 get_umr_flags(int acc)
{ … }

static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
			     struct mlx5_ib_mr *mr,
			     u32 key, int access)
{ … }

static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
{ … }

static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
			     struct mlx5_ib_mr *mr,
			     struct mlx5_ib_pd *pd)
{ … }

static __be32 send_ieth(const struct ib_send_wr *wr)
{ … }

static u8 calc_sig(void *wqe, int size)
{ … }

static u8 wq_sig(void *wqe)
{ … }

static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
			    void **wqe, int *wqe_sz, void **cur_edge)
{ … }

static u16 prot_field_size(enum ib_signature_type type)
{ … }

static u8 bs_selector(int block_size)
{ … }

static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
			      struct mlx5_bsf_inl *inl)
{ … }

static int mlx5_set_bsf(struct ib_mr *sig_mr,
			struct ib_sig_attrs *sig_attrs,
			struct mlx5_bsf *bsf, u32 data_size)
{ … }


static int set_sig_data_segment(const struct ib_send_wr *send_wr,
				struct ib_mr *sig_mr,
				struct ib_sig_attrs *sig_attrs,
				struct mlx5_ib_qp *qp, void **seg, int *size,
				void **cur_edge)
{ … }

static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
				 struct ib_mr *sig_mr, int access_flags,
				 u32 size, u32 length, u32 pdn)
{ … }

static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
				u32 size)
{ … }

static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
			 struct mlx5_ib_qp *qp, void **seg, int *size,
			 void **cur_edge)
{ … }

static int set_psv_wr(struct ib_sig_domain *domain,
		      u32 psv_idx, void **seg, int *size)
{ … }

static int set_reg_wr(struct mlx5_ib_qp *qp,
		      const struct ib_reg_wr *wr,
		      void **seg, int *size, void **cur_edge,
		      bool check_not_free)
{ … }

static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
			void **cur_edge)
{ … }

static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
{ … }

int mlx5r_begin_wqe(struct mlx5_ib_qp *qp, void **seg,
		    struct mlx5_wqe_ctrl_seg **ctrl, unsigned int *idx,
		    int *size, void **cur_edge, int nreq, __be32 general_id,
		    bool send_signaled, bool solicited)
{ … }

static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
		     struct mlx5_wqe_ctrl_seg **ctrl,
		     const struct ib_send_wr *wr, unsigned int *idx, int *size,
		     void **cur_edge, int nreq)
{ … }

void mlx5r_finish_wqe(struct mlx5_ib_qp *qp, struct mlx5_wqe_ctrl_seg *ctrl,
		      void *seg, u8 size, void *cur_edge, unsigned int idx,
		      u64 wr_id, int nreq, u8 fence, u32 mlx5_opcode)
{ … }

static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
{ … }

static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
			     struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
			     int *size, void **cur_edge, unsigned int idx)
{ … }

static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
			 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
			 void **cur_edge, unsigned int idx)
{ … }

static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
		      const struct ib_send_wr *wr,
		      struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
		      void **cur_edge, unsigned int *idx, int nreq,
		      struct ib_sig_domain *domain, u32 psv_index,
		      u8 next_fence)
{ … }

static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
				   struct mlx5_ib_qp *qp,
				   const struct ib_send_wr *wr,
				   struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
				   int *size, void **cur_edge,
				   unsigned int *idx, int nreq, u8 fence,
				   u8 next_fence)
{ … }

static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
			 const struct ib_send_wr *wr,
			 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
			 void **cur_edge, unsigned int *idx, int nreq, u8 fence,
			 u8 next_fence, int *num_sge)
{ … }

static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
{ … }

static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
			      const struct ib_send_wr *wr, void **seg,
			      int *size, void **cur_edge)
{ … }

static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
			  void **seg, int *size, void **cur_edge)
{ … }

void mlx5r_ring_db(struct mlx5_ib_qp *qp, unsigned int nreq,
		   struct mlx5_wqe_ctrl_seg *ctrl)
{ … }

int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
		      const struct ib_send_wr **bad_wr, bool drain)
{ … }

static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
{ … }

int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
		      const struct ib_recv_wr **bad_wr, bool drain)
{ … }