/* QLogic qedr NIC Driver * Copyright (c) 2015-2016 QLogic Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and /or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include <linux/dma-mapping.h> #include <linux/crc32.h> #include <net/ip.h> #include <net/ipv6.h> #include <net/udp.h> #include <linux/iommu.h> #include <rdma/ib_verbs.h> #include <rdma/ib_user_verbs.h> #include <rdma/iw_cm.h> #include <rdma/ib_umem.h> #include <rdma/ib_addr.h> #include <rdma/ib_cache.h> #include <rdma/uverbs_ioctl.h> #include <linux/qed/common_hsi.h> #include "qedr_hsi_rdma.h" #include <linux/qed/qed_if.h> #include "qedr.h" #include "verbs.h" #include <rdma/qedr-abi.h> #include "qedr_roce_cm.h" #include "qedr_iw_cm.h" #define QEDR_SRQ_WQE_ELEM_SIZE … #define RDMA_MAX_SGE_PER_SRQ … #define RDMA_MAX_SRQ_WQE_SIZE … #define DB_ADDR_SHIFT(addr) … enum { … }; static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len) { … } int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { … } int qedr_iw_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *sgid) { … } int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) { … } int qedr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *udata) { … } static inline void get_link_speed_and_width(int speed, u16 *ib_speed, u8 *ib_width) { … } int qedr_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *attr) { … } int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { … } void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) { … } void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) { … } int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) { … } int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) { … } int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata) { … } static void qedr_free_pbl(struct qedr_dev *dev, struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) { … } #define MIN_FW_PBL_PAGE_SIZE … #define MAX_FW_PBL_PAGE_SIZE … #define NUM_PBES_ON_PAGE(_page_size) … #define MAX_PBES_ON_PAGE … #define MAX_PBES_TWO_LAYER … static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, struct qedr_pbl_info *pbl_info, gfp_t flags) { … } static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, struct qedr_pbl_info *pbl_info, u32 num_pbes, int two_layer_capable) { … } static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, struct qedr_pbl *pbl, struct qedr_pbl_info *pbl_info, u32 pg_shift) { … } static int qedr_db_recovery_add(struct qedr_dev *dev, void __iomem *db_addr, void *db_data, enum qed_db_rec_width db_width, enum qed_db_rec_space db_space) { … } static void qedr_db_recovery_del(struct qedr_dev *dev, void __iomem *db_addr, void *db_data) { … } static int qedr_copy_cq_uresp(struct qedr_dev *dev, struct qedr_cq *cq, struct ib_udata *udata, u32 db_offset) { … } static void consume_cqe(struct qedr_cq *cq) { … } static inline int qedr_align_cq_entries(int entries) { … } static int qedr_init_user_db_rec(struct ib_udata *udata, struct qedr_dev *dev, struct qedr_userq *q, bool requires_db_rec) { … } static inline int qedr_init_user_queue(struct ib_udata *udata, struct qedr_dev *dev, struct qedr_userq *q, u64 buf_addr, size_t buf_len, bool requires_db_rec, int access, int alloc_and_init) { … } static inline void qedr_init_cq_params(struct qedr_cq *cq, struct qedr_ucontext *ctx, struct qedr_dev *dev, int vector, int chain_entries, int page_cnt, u64 pbl_ptr, struct qed_rdma_create_cq_in_params *params) { … } static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) { … } int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { … } int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { … } #define QEDR_DESTROY_CQ_MAX_ITERATIONS … #define QEDR_DESTROY_CQ_ITER_DURATION … int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { … } static inline int get_gid_info_from_table(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct qed_rdma_modify_qp_in_params *qp_params) { … } static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { … } static int qedr_copy_srq_uresp(struct qedr_dev *dev, struct qedr_srq *srq, struct ib_udata *udata) { … } static void qedr_copy_rq_uresp(struct qedr_dev *dev, struct qedr_create_qp_uresp *uresp, struct qedr_qp *qp) { … } static void qedr_copy_sq_uresp(struct qedr_dev *dev, struct qedr_create_qp_uresp *uresp, struct qedr_qp *qp) { … } static int qedr_copy_qp_uresp(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_udata *udata, struct qedr_create_qp_uresp *uresp) { … } static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) { … } static void qedr_set_common_qp_params(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_pd *pd, struct ib_qp_init_attr *attrs) { … } static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) { … } static int qedr_check_srq_params(struct qedr_dev *dev, struct ib_srq_init_attr *attrs, struct ib_udata *udata) { … } static void qedr_free_srq_user_params(struct qedr_srq *srq) { … } static void qedr_free_srq_kernel_params(struct qedr_srq *srq) { … } static int qedr_init_srq_user_params(struct ib_udata *udata, struct qedr_srq *srq, struct qedr_create_srq_ureq *ureq, int access) { … } static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, struct qedr_dev *dev, struct ib_srq_init_attr *init_attr) { … } int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, struct ib_udata *udata) { … } int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { … } int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) { … } static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type) { … } static inline void qedr_init_common_qp_in_params(struct qedr_dev *dev, struct qedr_pd *pd, struct qedr_qp *qp, struct ib_qp_init_attr *attrs, bool fmr_and_reserved_lkey, struct qed_rdma_create_qp_in_params *params) { … } static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) { … } static inline void qedr_iwarp_populate_user_qp(struct qedr_dev *dev, struct qedr_qp *qp, struct qed_rdma_create_qp_out_params *out_params) { … } static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_ucontext *ctx, struct qedr_qp *qp) { … } static int qedr_create_user_qp(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_pd *ibpd, struct ib_udata *udata, struct ib_qp_init_attr *attrs) { … } static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) { … } static int qedr_roce_create_kernel_qp(struct qedr_dev *dev, struct qedr_qp *qp, struct qed_rdma_create_qp_in_params *in_params, u32 n_sq_elems, u32 n_rq_elems) { … } static int qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, struct qedr_qp *qp, struct qed_rdma_create_qp_in_params *in_params, u32 n_sq_elems, u32 n_rq_elems) { … } static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) { … } static int qedr_create_kernel_qp(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_pd *ibpd, struct ib_qp_init_attr *attrs) { … } static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, struct ib_udata *udata) { … } int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { … } static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) { … } static enum qed_roce_qp_state qedr_get_state_from_ibqp( enum ib_qp_state qp_state) { … } static int qedr_update_qp_state(struct qedr_dev *dev, struct qedr_qp *qp, enum qed_roce_qp_state cur_state, enum qed_roce_qp_state new_state) { … } int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { … } static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) { … } int qedr_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int attr_mask, struct ib_qp_init_attr *qp_init_attr) { … } int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { … } int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata) { … } int qedr_destroy_ah(struct ib_ah *ibah, u32 flags) { … } static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) { … } static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, size_t page_list_len, bool two_layered) { … } struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 usr_addr, int acc, struct ib_udata *udata) { … } int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) { … } static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) { … } struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) { … } static int qedr_set_page(struct ib_mr *ibmr, u64 addr) { … } static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) { … } int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { … } struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) { … } static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) { … } static int sge_data_len(struct ib_sge *sg_list, int num_sge) { … } static void swap_wqe_data64(u64 *p) { … } static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, struct qedr_qp *qp, u8 *wqe_size, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr, u8 *bits, u8 bit) { … } #define RQ_SGE_SET(sge, vaddr, vlength, vflags) … #define SRQ_HDR_SET(hdr, vwr_id, num_sge) … #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) … static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, const struct ib_send_wr *wr) { … } static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, struct qedr_qp *qp, struct rdma_sq_rdma_wqe_1st *rwqe, struct rdma_sq_rdma_wqe_2nd *rwqe2, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { … } static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, struct qedr_qp *qp, struct rdma_sq_send_wqe_1st *swqe, struct rdma_sq_send_wqe_2st *swqe2, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { … } static int qedr_prepare_reg(struct qedr_qp *qp, struct rdma_sq_fmr_wqe_1st *fwqe1, const struct ib_reg_wr *wr) { … } static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) { … } static inline bool qedr_can_post_send(struct qedr_qp *qp, const struct ib_send_wr *wr) { … } static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { … } int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { … } static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) { … } int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { … } int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { … } static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) { … } static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) { … } static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) { … } /* Return latest CQE (needs processing) */ static union rdma_cqe *get_cqe(struct qedr_cq *cq) { … } /* In fmr we need to increase the number of fmr completed counter for the fmr * algorithm determining whether we can free a pbl or not. * we need to perform this whether the work request was signaled or not. for * this purpose we call this function from the condition that checks if a wr * should be skipped, to make sure we don't miss it ( possibly this fmr * operation was not signalted) */ static inline void qedr_chk_if_fmr(struct qedr_qp *qp) { … } static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, int num_entries, struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, int force) { … } static int qedr_poll_cq_req(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, int num_entries, struct ib_wc *wc, struct rdma_cqe_requester *req) { … } static inline int qedr_cqe_resp_status_to_ib(u8 status) { … } static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, struct ib_wc *wc) { … } static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, struct ib_wc *wc, struct rdma_cqe_responder *resp, u64 wr_id) { … } static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, struct ib_wc *wc, struct rdma_cqe_responder *resp) { … } static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, struct ib_wc *wc, struct rdma_cqe_responder *resp) { … } static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, int num_entries, struct ib_wc *wc, u16 hw_cons) { … } static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, struct rdma_cqe_responder *resp, int *update) { … } static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, int num_entries, struct ib_wc *wc, struct rdma_cqe_responder *resp) { … } static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, struct qedr_cq *cq, int num_entries, struct ib_wc *wc, struct rdma_cqe_responder *resp, int *update) { … } static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, struct rdma_cqe_requester *req, int *update) { … } int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { … } int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, u32 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad *in, struct ib_mad *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index) { … }
Codebase Browser
linux