linux/drivers/infiniband/hw/mlx5/qp.c

/*
 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/etherdevice.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/rdma_counter.h>
#include <linux/mlx5/fs.h>
#include "mlx5_ib.h"
#include "ib_rep.h"
#include "counters.h"
#include "cmd.h"
#include "umr.h"
#include "qp.h"
#include "wr.h"

enum { … };

enum { … };

enum raw_qp_set_mask_map { … };

enum { … };

struct mlx5_modify_raw_qp_param { … };

struct mlx5_ib_qp_event_work { … };

static struct workqueue_struct *mlx5_ib_qp_event_wq;

static void get_cqs(enum ib_qp_type qp_type,
		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);

static int is_qp0(enum ib_qp_type qp_type)
{ … }

static int is_sqp(enum ib_qp_type qp_type)
{ … }

/**
 * mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
 * to kernel buffer
 *
 * @umem: User space memory where the WQ is
 * @buffer: buffer to copy to
 * @buflen: buffer length
 * @wqe_index: index of WQE to copy from
 * @wq_offset: offset to start of WQ
 * @wq_wqe_cnt: number of WQEs in WQ
 * @wq_wqe_shift: log2 of WQE size
 * @bcnt: number of bytes to copy
 * @bytes_copied: number of bytes to copy (return value)
 *
 * Copies from start of WQE bcnt or less bytes.
 * Does not gurantee to copy the entire WQE.
 *
 * Return: zero on success, or an error code.
 */
static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,
					size_t buflen, int wqe_index,
					int wq_offset, int wq_wqe_cnt,
					int wq_wqe_shift, int bcnt,
					size_t *bytes_copied)
{ … }

static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
				      void *buffer, size_t buflen, size_t *bc)
{ … }

static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
				    void *buffer, size_t buflen, size_t *bc)
{ … }

int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
			size_t buflen, size_t *bc)
{ … }

static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
				    void *buffer, size_t buflen, size_t *bc)
{ … }

int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
			size_t buflen, size_t *bc)
{ … }

static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
				     void *buffer, size_t buflen, size_t *bc)
{ … }

int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
			 size_t buflen, size_t *bc)
{ … }

static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
{ … }

static void mlx5_ib_handle_qp_event(struct work_struct *_work)
{ … }

static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
{ … }

static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
		       int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
{ … }

static int sq_overhead(struct ib_qp_init_attr *attr)
{ … }

static int calc_send_wqe(struct ib_qp_init_attr *attr)
{ … }

static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
{ … }

static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
			struct mlx5_ib_qp *qp)
{ … }

static int set_user_buf_size(struct mlx5_ib_dev *dev,
			    struct mlx5_ib_qp *qp,
			    struct mlx5_ib_create_qp *ucmd,
			    struct mlx5_ib_qp_base *base,
			    struct ib_qp_init_attr *attr)
{ … }

static int qp_has_rq(struct ib_qp_init_attr *attr)
{ … }

enum { … };

static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
{ … }

static int num_med_bfreg(struct mlx5_ib_dev *dev,
			 struct mlx5_bfreg_info *bfregi)
{ … }

static int first_med_bfreg(struct mlx5_ib_dev *dev,
			   struct mlx5_bfreg_info *bfregi)
{ … }

static int first_hi_bfreg(struct mlx5_ib_dev *dev,
			  struct mlx5_bfreg_info *bfregi)
{ … }

static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
				  struct mlx5_bfreg_info *bfregi)
{ … }

static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
				 struct mlx5_bfreg_info *bfregi)
{ … }

static int alloc_bfreg(struct mlx5_ib_dev *dev,
		       struct mlx5_bfreg_info *bfregi)
{ … }

void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
{ … }

static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
{ … }

static int to_mlx5_st(enum ib_qp_type type)
{ … }

static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
			     struct mlx5_ib_cq *recv_cq);
static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
			       struct mlx5_ib_cq *recv_cq);

int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
			struct mlx5_bfreg_info *bfregi, u32 bfregn,
			bool dyn_bfreg)
{ … }

static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			    struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
{ … }

static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			  struct ib_udata *udata, struct mlx5_ib_rwq *rwq,
			  struct mlx5_ib_create_wq *ucmd)
{ … }

static int adjust_bfregn(struct mlx5_ib_dev *dev,
			 struct mlx5_bfreg_info *bfregi, int bfregn)
{ … }

static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			   struct mlx5_ib_qp *qp, struct ib_udata *udata,
			   struct ib_qp_init_attr *attr, u32 **in,
			   struct mlx5_ib_create_qp_resp *resp, int *inlen,
			   struct mlx5_ib_qp_base *base,
			   struct mlx5_ib_create_qp *ucmd)
{ … }

static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
		       struct mlx5_ib_qp_base *base, struct ib_udata *udata)
{ … }

static int _create_kernel_qp(struct mlx5_ib_dev *dev,
			     struct ib_qp_init_attr *init_attr,
			     struct mlx5_ib_qp *qp, u32 **in, int *inlen,
			     struct mlx5_ib_qp_base *base)
{ … }

static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
{ … }

static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
				    struct mlx5_ib_qp *qp,
				    struct mlx5_ib_sq *sq, u32 tdn,
				    struct ib_pd *pd)
{ … }

static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
				      struct mlx5_ib_sq *sq, struct ib_pd *pd)
{ … }

static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
{ … }

static bool fr_supported(int ts_cap)
{ … }

static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
			 bool fr_sup, bool rt_sup)
{ … }

static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq)
{ … }

static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq)
{ … }

static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq,
			    struct mlx5_ib_cq *recv_cq)
{ … }

static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
				   struct ib_udata *udata,
				   struct mlx5_ib_sq *sq, void *qpin,
				   struct ib_pd *pd, struct mlx5_ib_cq *cq)
{ … }

static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
				     struct mlx5_ib_sq *sq)
{ … }

static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
				   struct mlx5_ib_rq *rq, void *qpin,
				   struct ib_pd *pd, struct mlx5_ib_cq *cq)
{ … }

static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
				     struct mlx5_ib_rq *rq)
{ … }

static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
				      struct mlx5_ib_rq *rq,
				      u32 qp_flags_en,
				      struct ib_pd *pd)
{ … }

static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
				    struct mlx5_ib_rq *rq, u32 tdn,
				    u32 *qp_flags_en, struct ib_pd *pd,
				    u32 *out)
{ … }

static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
				u32 *in, size_t inlen, struct ib_pd *pd,
				struct ib_udata *udata,
				struct mlx5_ib_create_qp_resp *resp,
				struct ib_qp_init_attr *init_attr)
{ … }

static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
				  struct mlx5_ib_qp *qp)
{ … }

static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
				    struct mlx5_ib_raw_packet_qp *raw_packet_qp)
{ … }

static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
{ … }

struct mlx5_create_qp_params { … };

static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
				 struct mlx5_ib_qp *qp,
				 struct mlx5_create_qp_params *params)
{ … }

static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
					 struct mlx5_ib_qp *qp,
					 struct ib_qp_init_attr *init_attr,
					 void *qpc)
{ … }

static int atomic_size_to_mode(int size_mask)
{ … }

static int get_atomic_mode(struct mlx5_ib_dev *dev,
			   enum ib_qp_type qp_type)
{ … }

static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
			     struct mlx5_create_qp_params *params)
{ … }

static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
		      struct mlx5_ib_qp *qp,
		      struct mlx5_create_qp_params *params)
{ … }

static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			  struct mlx5_ib_qp *qp,
			  struct mlx5_create_qp_params *params)
{ … }

static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			    struct mlx5_ib_qp *qp,
			    struct mlx5_create_qp_params *params)
{ … }

static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
{ … }

static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
	__releases(&send_cq->lock) __releases(&recv_cq->lock)
{ … }

static void get_cqs(enum ib_qp_type qp_type,
		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
{ … }

static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
				const struct mlx5_modify_raw_qp_param *raw_qp_param,
				u8 lag_tx_affinity);

static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
			      struct ib_udata *udata)
{ … }

static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,
		      struct mlx5_ib_qp *qp,
		      struct mlx5_create_qp_params *params)
{ … }

static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
			 enum ib_qp_type *type)
{ … }

static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			    struct ib_qp_init_attr *attr,
			    struct ib_udata *udata)
{ … }

static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
				bool cond, struct mlx5_ib_qp *qp)
{ … }

static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
				void *ucmd, struct ib_qp_init_attr *attr)
{ … }

static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
				bool cond, struct mlx5_ib_qp *qp)
{ … }

static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
				struct ib_qp_init_attr *attr)
{ … }

static int process_udata_size(struct mlx5_ib_dev *dev,
			      struct mlx5_create_qp_params *params)
{ … }

static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
		     struct mlx5_ib_qp *qp,
		     struct mlx5_create_qp_params *params)
{ … }

static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
			 struct ib_qp_init_attr *attr)
{ … }

static int get_qp_uidx(struct mlx5_ib_qp *qp,
		       struct mlx5_create_qp_params *params)
{ … }

static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
{ … }

static int check_ucmd_data(struct mlx5_ib_dev *dev,
			   struct mlx5_create_qp_params *params)
{ … }

int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
		      struct ib_udata *udata)
{ … }

int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
{ … }

static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
				const struct ib_qp_attr *attr, int attr_mask,
				void *qpc)
{ … }

enum { … };

static int mlx5_to_ib_rate_map(u8 rate)
{ … }

static int ib_to_mlx5_rate_map(u8 rate)
{ … }

static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
{ … }

static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
				      struct mlx5_ib_sq *sq, u8 sl,
				      struct ib_pd *pd)
{ … }

static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
					 struct mlx5_ib_sq *sq, u8 tx_affinity,
					 struct ib_pd *pd)
{ … }

static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,
				    u32 lqpn, u32 rqpn)

{ … }

static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
			 const struct rdma_ah_attr *ah, void *path, u8 port,
			 int attr_mask, u32 path_flags,
			 const struct ib_qp_attr *attr, bool alt)
{ … }

static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = …;

static int ib_nr_to_mlx5_nr(int ib_mask)
{ … }

static int ib_mask_to_mlx5_opt(int ib_mask)
{ … }

static int modify_raw_packet_qp_rq(
	struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
{ … }

static int modify_raw_packet_qp_sq(
	struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state,
	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
{ … }

static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
				const struct mlx5_modify_raw_qp_param *raw_qp_param,
				u8 tx_affinity)
{ … }

static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
				       struct ib_udata *udata)
{ … }

static bool qp_supports_affinity(struct mlx5_ib_qp *qp)
{ … }

static unsigned int get_tx_affinity(struct ib_qp *qp,
				    const struct ib_qp_attr *attr,
				    int attr_mask, u8 init,
				    struct ib_udata *udata)
{ … }

static int __mlx5_ib_qp_set_raw_qp_counter(struct mlx5_ib_qp *qp, u32 set_id,
					   struct mlx5_core_dev *mdev)
{ … }

static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
				    struct rdma_counter *counter)
{ … }

static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
			       const struct ib_qp_attr *attr, int attr_mask,
			       enum ib_qp_state cur_state,
			       enum ib_qp_state new_state,
			       const struct mlx5_ib_modify_qp *ucmd,
			       struct mlx5_ib_modify_qp_resp *resp,
			       struct ib_udata *udata)
{ … }

static inline bool is_valid_mask(int mask, int req, int opt)
{ … }

/* check valid transition for driver QP types
 * for now the only QP type that this function supports is DCI
 */
static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
				enum ib_qp_attr_mask attr_mask)
{ … }

/* mlx5_ib_modify_dct: modify a DCT QP
 * valid transitions are:
 * RESET to INIT: must set access_flags, pkey_index and port
 * INIT  to RTR : must set min_rnr_timer, tclass, flow_label,
 *			   mtu, gid_index and hop_limit
 * Other transitions and attributes are illegal
 */
static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
			      int attr_mask, struct mlx5_ib_modify_qp *ucmd,
			      struct ib_udata *udata)
{ … }

static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,
				      struct mlx5_ib_qp *qp)
{ … }

static int validate_rd_atomic(struct mlx5_ib_dev *dev, struct ib_qp_attr *attr,
			      int attr_mask, enum ib_qp_type qp_type)
{ … }

int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
		      int attr_mask, struct ib_udata *udata)
{ … }

static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
{ … }

static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
{ … }

static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
			    struct rdma_ah_attr *ah_attr, void *path)
{ … }

static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
					struct mlx5_ib_sq *sq,
					u8 *sq_state)
{ … }

static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
					struct mlx5_ib_rq *rq,
					u8 *rq_state)
{ … }

static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
				  struct mlx5_ib_qp *qp, u8 *qp_state)
{ … }

static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
				     struct mlx5_ib_qp *qp,
				     u8 *raw_packet_qp_state)
{ … }

static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
			 struct ib_qp_attr *qp_attr)
{ … }

static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
				struct ib_qp_attr *qp_attr, int qp_attr_mask,
				struct ib_qp_init_attr *qp_init_attr)
{ … }

int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
{ … }

int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
{ … }

int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
{ … }

static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
{ … }

static int set_delay_drop(struct mlx5_ib_dev *dev)
{ … }

static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
		      struct ib_wq_init_attr *init_attr)
{ … }

static int set_user_rq_size(struct mlx5_ib_dev *dev,
			    struct ib_wq_init_attr *wq_init_attr,
			    struct mlx5_ib_create_wq *ucmd,
			    struct mlx5_ib_rwq *rwq)
{ … }

static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
{ … }

static int prepare_user_rq(struct ib_pd *pd,
			   struct ib_wq_init_attr *init_attr,
			   struct ib_udata *udata,
			   struct mlx5_ib_rwq *rwq)
{ … }

struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
				struct ib_wq_init_attr *init_attr,
				struct ib_udata *udata)
{ … }

int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
{ … }

int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
				 struct ib_rwq_ind_table_init_attr *init_attr,
				 struct ib_udata *udata)
{ … }

int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
{ … }

int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
		      u32 wq_attr_mask, struct ib_udata *udata)
{ … }

struct mlx5_ib_drain_cqe { … };

static void mlx5_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
{ … }

/* This function returns only once the drained WR was completed */
static void handle_drain_completion(struct ib_cq *cq,
				    struct mlx5_ib_drain_cqe *sdrain,
				    struct mlx5_ib_dev *dev)
{ … }

void mlx5_ib_drain_sq(struct ib_qp *qp)
{ … }

void mlx5_ib_drain_rq(struct ib_qp *qp)
{ … }

/*
 * Bind a qp to a counter. If @counter is NULL then bind the qp to
 * the default counter
 */
int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)
{ … }

int mlx5_ib_qp_event_init(void)
{ … }

void mlx5_ib_qp_event_cleanup(void)
{ … }