/* This file is part of the Emulex RoCE Device Driver for * RoCE (RDMA over Converged Ethernet) adapters. * Copyright (C) 2012-2015 Emulex. All rights reserved. * EMULEX and SLI are trademarks of Emulex. * www.emulex.com * * This software is available to you under a choice of one of two licenses. * You may choose to be licensed under the terms of the GNU General Public * License (GPL) Version 2, available from the file COPYING in the main * directory of this source tree, or the BSD license below: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * - Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Contact Information: * [email protected] * * Emulex * 3333 Susan Street * Costa Mesa, CA 92626 */ #include <linux/dma-mapping.h> #include <net/addrconf.h> #include <rdma/ib_verbs.h> #include <rdma/ib_user_verbs.h> #include <rdma/iw_cm.h> #include <rdma/ib_umem.h> #include <rdma/ib_addr.h> #include <rdma/ib_cache.h> #include <rdma/uverbs_ioctl.h> #include "ocrdma.h" #include "ocrdma_hw.h" #include "ocrdma_verbs.h" #include <rdma/ocrdma-abi.h> int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { … } int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *uhw) { … } static inline void get_link_speed_and_width(struct ocrdma_dev *dev, u16 *ib_speed, u8 *ib_width) { … } int ocrdma_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { … } static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, unsigned long len) { … } static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, unsigned long len) { … } static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr, unsigned long len) { … } static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool) { … } static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id, bool dpp_pool) { … } static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id, bool dpp_pool) { … } static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd) { … } /* * NOTE: * * ocrdma_ucontext must be used here because this function is also * called from ocrdma_alloc_ucontext where ib_udata does not have * valid ib_ucontext pointer. ib_uverbs_get_context does not call * uobj_{alloc|get_xxx} helpers which are used to store the * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so * ib_udata does NOT imply valid ib_ucontext here! */ static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd, struct ocrdma_ucontext *uctx, struct ib_udata *udata) { … } static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx, struct ocrdma_pd *pd) { … } static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd) { … } static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev, struct ocrdma_ucontext *uctx, struct ib_udata *udata) { … } static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx) { … } static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx) { … } static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx) { … } int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { … } void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) { … } int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) { … } static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, struct ib_udata *udata) { … } int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr, u32 pdid, int acc, u32 num_pbls, u32 addr_check) { … } struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) { … } static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) { … } static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr, u32 num_pbes) { … } static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr) { … } static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr) { … } struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 usr_addr, int acc, struct ib_udata *udata) { … } int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) { … } static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq, struct ib_udata *udata) { … } int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { … } int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) { … } static void ocrdma_flush_cq(struct ocrdma_cq *cq) { … } int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { … } static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) { … } static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) { … } static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { … } static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp, struct ib_udata *udata, int dpp_offset, int dpp_credit_lmt, int srq) { … } static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp, struct ocrdma_pd *pd) { … } static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp) { … } static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp, struct ocrdma_pd *pd, struct ib_qp_init_attr *attrs) { … } static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev, struct ib_qp_init_attr *attrs) { … } int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { … } int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask) { … } int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { … } static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu) { … } static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags) { … } int ocrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int attr_mask, struct ib_qp_init_attr *qp_init_attr) { … } static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx) { … } static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) { … } static int is_hw_sq_empty(struct ocrdma_qp *qp) { … } static int is_hw_rq_empty(struct ocrdma_qp *qp) { … } static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) { … } static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q, u32 idx) { … } static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q) { … } static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q) { … } /* discard the cqe for a given QP */ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) { … } void ocrdma_del_flush_qp(struct ocrdma_qp *qp) { … } int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { … } static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq, struct ib_udata *udata) { … } int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, struct ib_udata *udata) { … } int ocrdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata) { … } int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) { … } int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { … } /* unprivileged verbs and their support functions. */ static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, const struct ib_send_wr *wr) { … } static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr, struct ocrdma_sge *sge, int num_sge, struct ib_sge *sg_list) { … } static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge) { … } static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, struct ocrdma_sge *sge, const struct ib_send_wr *wr, u32 wqe_size) { … } static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, const struct ib_send_wr *wr) { … } static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, const struct ib_send_wr *wr) { … } static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, const struct ib_send_wr *wr) { … } static int get_encoded_page_size(int pg_sz) { … } static int ocrdma_build_reg(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr, const struct ib_reg_wr *wr) { … } static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) { … } int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { … } static void ocrdma_ring_rq_db(struct ocrdma_qp *qp) { … } static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, const struct ib_recv_wr *wr, u16 tag) { … } int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { … } /* cqe for srq's rqe can potentially arrive out of order. * index gives the entry in the shadow table where to store * the wr_id. tag/index is returned in cqe to reference back * for a given rqe. */ static int ocrdma_srq_get_idx(struct ocrdma_srq *srq) { … } static void ocrdma_ring_srq_db(struct ocrdma_srq *srq) { … } int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { … } static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) { … } static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc, u32 wqe_idx) { … } static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe) { … } static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, struct ocrdma_qp *qp, int status) { … } static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, struct ocrdma_qp *qp, int status) { … } static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, struct ocrdma_qp *qp, int status) { … } static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, struct ib_wc *ibwc, bool *polled, bool *stop) { … } static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, struct ib_wc *ibwc, bool *polled) { … } static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, struct ib_wc *ibwc, bool *polled, bool *stop) { … } static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc, struct ocrdma_cqe *cqe) { … } static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe, struct ocrdma_qp *qp) { … } static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, struct ib_wc *ibwc, bool *polled, bool *stop, int status) { … } static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, struct ib_wc *ibwc) { … } static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe, struct ib_wc *ibwc, bool *polled, bool *stop) { … } static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe, u16 cur_getp) { … } static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, struct ib_wc *ibwc) { … } /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */ static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries, struct ocrdma_qp *qp, struct ib_wc *ibwc) { … } int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { … } int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) { … } struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) { … } static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr) { … } int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { … }
Codebase Browser
linux