erdma_verbs.c | Explore in Territory

// SPDX-License-Identifier: GPL-2.0

/* Authors: Cheng Xu <[email protected]> */
/*          Kai Shen <[email protected]> */
/* Copyright (c) 2020-2022, Alibaba Group. */

/* Authors: Bernard Metzler <[email protected]> */
/* Copyright (c) 2008-2019, IBM Corporation */

/* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */

#include <linux/vmalloc.h>
#include <net/addrconf.h>
#include <rdma/erdma-abi.h>
#include <rdma/ib_umem.h>
#include <rdma/uverbs_ioctl.h>

#include "erdma.h"
#include "erdma_cm.h"
#include "erdma_verbs.h"

static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
				      u64 *addr0, u64 *addr1)
{ … }

static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
{ … }

static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
{ … }

static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
{ … }

static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
{ … }

static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
{ … }

static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
			     u32 size, u8 mmap_flag, u64 *mmap_offset)
{ … }

int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
		       struct ib_udata *unused)
{ … }

int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
		    union ib_gid *gid)
{ … }

int erdma_query_port(struct ib_device *ibdev, u32 port,
		     struct ib_port_attr *attr)
{ … }

int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
			     struct ib_port_immutable *port_immutable)
{ … }

int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{ … }

int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{ … }

static void erdma_flush_worker(struct work_struct *work)
{ … }

static int erdma_qp_validate_cap(struct erdma_dev *dev,
				 struct ib_qp_init_attr *attrs)
{ … }

static int erdma_qp_validate_attr(struct erdma_dev *dev,
				  struct ib_qp_init_attr *attrs)
{ … }

static void free_kernel_qp(struct erdma_qp *qp)
{ … }

static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
			  struct ib_qp_init_attr *attrs)
{ … }

static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
{ … }

static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
					       size_t size)
{ … }

static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,
				     struct erdma_mtt *mtt)
{ … }

static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
				      struct erdma_mtt *mtt)
{ … }

static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
				  struct erdma_mtt *low_mtt)
{ … }

static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt)
{ … }

static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
						  size_t size)
{ … }

static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
					  bool force_continuous)
{ … }

static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
{ … }

static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
			   u64 start, u64 len, int access, u64 virt,
			   unsigned long req_page_size, bool force_continuous)
{ … }

static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
{ … }

static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
				    u64 dbrecords_va,
				    struct erdma_user_dbrecords_page **dbr_page,
				    dma_addr_t *dma_addr)
{ … }

static void
erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
			   struct erdma_user_dbrecords_page **dbr_page)
{ … }

static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
			u64 va, u32 len, u64 dbrec_va)
{ … }

static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
{ … }

int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
		    struct ib_udata *udata)
{ … }

static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
{ … }

struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
{ … }

struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
				u32 max_num_sg)
{ … }

static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
{ … }

int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
		    unsigned int *sg_offset)
{ … }

struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
				u64 virt, int access, struct ib_udata *udata)
{ … }

int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
{ … }

int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{ … }

int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
{ … }

void erdma_qp_get_ref(struct ib_qp *ibqp)
{ … }

void erdma_qp_put_ref(struct ib_qp *ibqp)
{ … }

int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
{ … }

void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
{ … }

static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
			      bool ext_db_en)
{ … }

static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
{ … }

static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
{ … }

int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
{ … }

void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
{ … }

static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = …;

int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
		    struct ib_udata *udata)
{ … }

static enum ib_qp_state query_qp_state(struct erdma_qp *qp)
{ … }

int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
		   int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
{ … }

static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
			      struct erdma_ureq_create_cq *ureq)
{ … }

static int erdma_init_kernel_cq(struct erdma_cq *cq)
{ … }

int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
		    struct uverbs_attr_bundle *attrs)
{ … }

void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)
{ … }

void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
{ … }

void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
{ … }

enum counters { … };

static const struct rdma_stat_desc erdma_descs[] = …;

struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
						u32 port_num)
{ … }

static int erdma_query_hw_stats(struct erdma_dev *dev,
				struct rdma_hw_stats *stats)
{ … }

int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
		       u32 port, int index)
{ … }
linux/drivers/infiniband/hw/erdma/erdma_verbs.c
