// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include <linux/dma-buf.h> #include <linux/dma-resv.h> #include <linux/vmalloc.h> #include <linux/log2.h> #include <rdma/ib_addr.h> #include <rdma/ib_umem.h> #include <rdma/ib_user_verbs.h> #include <rdma/ib_verbs.h> #include <rdma/uverbs_ioctl.h> #define UVERBS_MODULE_NAME … #include <rdma/uverbs_named_ioctl.h> #include <rdma/ib_user_ioctl_cmds.h> #include "efa.h" #include "efa_io_defs.h" enum { … }; struct efa_user_mmap_entry { … }; #define EFA_DEFINE_DEVICE_STATS(op) … #define EFA_DEFINE_PORT_STATS(op) … \ #define EFA_STATS_ENUM(ename, name) … #define EFA_STATS_STR(ename, nam) … enum efa_hw_device_stats { … }; static const struct rdma_stat_desc efa_device_stats_descs[] = …; enum efa_hw_port_stats { … }; static const struct rdma_stat_desc efa_port_stats_descs[] = …; #define EFA_CHUNK_PAYLOAD_SHIFT … #define EFA_CHUNK_PAYLOAD_SIZE … #define EFA_CHUNK_PAYLOAD_PTR_SIZE … #define EFA_CHUNK_SHIFT … #define EFA_CHUNK_SIZE … #define EFA_CHUNK_PTR_SIZE … #define EFA_PTRS_PER_CHUNK … #define EFA_CHUNK_USED_SIZE … struct pbl_chunk { … }; struct pbl_chunk_list { … }; struct pbl_context { … }; static inline struct efa_dev *to_edev(struct ib_device *ibdev) { … } static inline struct efa_ucontext *to_eucontext(struct ib_ucontext *ibucontext) { … } static inline struct efa_pd *to_epd(struct ib_pd *ibpd) { … } static inline struct efa_mr *to_emr(struct ib_mr *ibmr) { … } static inline struct efa_qp *to_eqp(struct ib_qp *ibqp) { … } static inline struct efa_cq *to_ecq(struct ib_cq *ibcq) { … } static inline struct efa_ah *to_eah(struct ib_ah *ibah) { … } static inline struct efa_user_mmap_entry * to_emmap(struct rdma_user_mmap_entry *rdma_entry) { … } #define EFA_DEV_CAP(dev, cap) … #define is_reserved_cleared(reserved) … static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr, size_t size, enum dma_data_direction dir) { … } static void efa_free_mapped(struct efa_dev *dev, void *cpu_addr, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { … } int efa_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct ib_udata *udata) { … } int efa_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { … } int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) { … } int efa_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { … } int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) { … } static int efa_pd_dealloc(struct efa_dev *dev, u16 pdn) { … } int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { … } static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle) { … } static void efa_qp_user_mmap_entries_remove(struct efa_qp *qp) { … } int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { … } static struct rdma_user_mmap_entry* efa_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length, u8 mmap_flag, u64 *offset) { … } static int qp_mmap_entries_setup(struct efa_qp *qp, struct efa_dev *dev, struct efa_ucontext *ucontext, struct efa_com_create_qp_params *params, struct efa_ibv_create_qp_resp *resp) { … } static int efa_qp_validate_cap(struct efa_dev *dev, struct ib_qp_init_attr *init_attr) { … } static int efa_qp_validate_attr(struct efa_dev *dev, struct ib_qp_init_attr *init_attr) { … } int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { … } static const struct { … } srd_qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = …; static bool efa_modify_srd_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, enum ib_qp_attr_mask mask) { … } static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, enum ib_qp_state cur_state, enum ib_qp_state new_state) { … } int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata) { … } static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx) { … } static void efa_cq_user_mmap_entries_remove(struct efa_cq *cq) { … } int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { … } static struct efa_eq *efa_vec2eq(struct efa_dev *dev, int vec) { … } static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, struct efa_ibv_create_cq_resp *resp, bool db_valid) { … } int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { … } static int umem_to_page_list(struct efa_dev *dev, struct ib_umem *umem, u64 *page_list, u32 hp_cnt, u8 hp_shift) { … } static struct scatterlist *efa_vmalloc_buf_to_sg(u64 *buf, int page_cnt) { … } /* * create a chunk list of physical pages dma addresses from the supplied * scatter gather list */ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) { … } static void pbl_chunk_list_destroy(struct efa_dev *dev, struct pbl_context *pbl) { … } /* initialize pbl continuous mode: map pbl buffer to a dma address. */ static int pbl_continuous_initialize(struct efa_dev *dev, struct pbl_context *pbl) { … } /* * initialize pbl indirect mode: * create a chunk list out of the dma addresses of the physical pages of * pbl buffer. */ static int pbl_indirect_initialize(struct efa_dev *dev, struct pbl_context *pbl) { … } static void pbl_indirect_terminate(struct efa_dev *dev, struct pbl_context *pbl) { … } /* create a page buffer list from a mapped user memory region */ static int pbl_create(struct efa_dev *dev, struct pbl_context *pbl, struct ib_umem *umem, int hp_cnt, u8 hp_shift) { … } static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl) { … } static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr, struct efa_com_reg_mr_params *params) { … } static int efa_create_pbl(struct efa_dev *dev, struct pbl_context *pbl, struct efa_mr *mr, struct efa_com_reg_mr_params *params) { … } static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, struct ib_udata *udata) { … } static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start, u64 length, u64 virt_addr, int access_flags) { … } struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int fd, int access_flags, struct uverbs_attr_bundle *attrs) { … } struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata) { … } static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *attrs) { … } int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { … } int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { … } static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn) { … } #define EFA_CHECK_USER_COMP(_dev, _comp_mask, _attr, _mask, _attr_str) … static int efa_user_comp_handshake(const struct ib_ucontext *ibucontext, const struct efa_ibv_alloc_ucontext_cmd *cmd) { … } int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) { … } void efa_dealloc_ucontext(struct ib_ucontext *ibucontext) { … } void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry) { … } static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, struct vm_area_struct *vma) { … } int efa_mmap(struct ib_ucontext *ibucontext, struct vm_area_struct *vma) { … } static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah) { … } int efa_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata) { … } int efa_destroy_ah(struct ib_ah *ibah, u32 flags) { … } struct rdma_hw_stats *efa_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num) { … } struct rdma_hw_stats *efa_alloc_hw_device_stats(struct ib_device *ibdev) { … } static int efa_fill_device_stats(struct efa_dev *dev, struct rdma_hw_stats *stats) { … } static int efa_fill_port_stats(struct efa_dev *dev, struct rdma_hw_stats *stats, u32 port_num) { … } int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, u32 port_num, int index) { … } enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, u32 port_num) { … } DECLARE_UVERBS_NAMED_METHOD(…); ADD_UVERBS_METHODS(efa_mr, UVERBS_OBJECT_MR, &UVERBS_METHOD(EFA_IB_METHOD_MR_QUERY)); const struct uapi_definition efa_uapi_defs[] = …;
Codebase Browser
linux