// SPDX-License-Identifier: GPL-2.0-only /* * Huawei HiNIC PCI Express Linux driver * Copyright(c) 2017 Huawei Technologies Co., Ltd */ #include <linux/kernel.h> #include <linux/types.h> #include <linux/pci.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/vmalloc.h> #include <linux/errno.h> #include <linux/sizes.h> #include <linux/atomic.h> #include <linux/skbuff.h> #include <linux/io.h> #include <asm/barrier.h> #include <asm/byteorder.h> #include "hinic_common.h" #include "hinic_hw_if.h" #include "hinic_hw_wqe.h" #include "hinic_hw_wq.h" #include "hinic_hw_qp_ctxt.h" #include "hinic_hw_qp.h" #include "hinic_hw_io.h" #define SQ_DB_OFF … /* The number of cache line to prefetch Until threshold state */ #define WQ_PREFETCH_MAX … /* The number of cache line to prefetch After threshold state */ #define WQ_PREFETCH_MIN … /* Threshold state */ #define WQ_PREFETCH_THRESHOLD … /* sizes of the SQ/RQ ctxt */ #define Q_CTXT_SIZE … #define CTXT_RSVD … #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) … #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) … #define SIZE_16BYTES(size) … #define SIZE_8BYTES(size) … #define SECT_SIZE_FROM_8BYTES(size) … #define SQ_DB_PI_HI_SHIFT … #define SQ_DB_PI_HI(prod_idx) … #define SQ_DB_PI_LOW_MASK … #define SQ_DB_PI_LOW(prod_idx) … #define SQ_DB_ADDR(sq, pi) … #define SQ_MASKED_IDX(sq, idx) … #define RQ_MASKED_IDX(rq, idx) … enum sq_wqe_type { … }; enum rq_completion_fmt { … }; void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr, enum hinic_qp_ctxt_type ctxt_type, u16 num_queues, u16 max_queues) { … } void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt, struct hinic_sq *sq, u16 global_qid) { … } void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt, struct hinic_rq *rq, u16 global_qid) { … } /** * alloc_sq_skb_arr - allocate sq array for saved skb * @sq: HW Send Queue * * Return 0 - Success, negative - Failure **/ static int alloc_sq_skb_arr(struct hinic_sq *sq) { … } /** * free_sq_skb_arr - free sq array for saved skb * @sq: HW Send Queue **/ static void free_sq_skb_arr(struct hinic_sq *sq) { … } /** * alloc_rq_skb_arr - allocate rq array for saved skb * @rq: HW Receive Queue * * Return 0 - Success, negative - Failure **/ static int alloc_rq_skb_arr(struct hinic_rq *rq) { … } /** * free_rq_skb_arr - free rq array for saved skb * @rq: HW Receive Queue **/ static void free_rq_skb_arr(struct hinic_rq *rq) { … } /** * hinic_init_sq - Initialize HW Send Queue * @sq: HW Send Queue * @hwif: HW Interface for accessing HW * @wq: Work Queue for the data of the SQ * @entry: msix entry for sq * @ci_addr: address for reading the current HW consumer index * @ci_dma_addr: dma address for reading the current HW consumer index * @db_base: doorbell base address * * Return 0 - Success, negative - Failure **/ int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif, struct hinic_wq *wq, struct msix_entry *entry, void *ci_addr, dma_addr_t ci_dma_addr, void __iomem *db_base) { … } /** * hinic_clean_sq - Clean HW Send Queue's Resources * @sq: Send Queue **/ void hinic_clean_sq(struct hinic_sq *sq) { … } /** * alloc_rq_cqe - allocate rq completion queue elements * @rq: HW Receive Queue * * Return 0 - Success, negative - Failure **/ static int alloc_rq_cqe(struct hinic_rq *rq) { … } /** * free_rq_cqe - free rq completion queue elements * @rq: HW Receive Queue **/ static void free_rq_cqe(struct hinic_rq *rq) { … } /** * hinic_init_rq - Initialize HW Receive Queue * @rq: HW Receive Queue * @hwif: HW Interface for accessing HW * @wq: Work Queue for the data of the RQ * @entry: msix entry for rq * * Return 0 - Success, negative - Failure **/ int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif, struct hinic_wq *wq, struct msix_entry *entry) { … } /** * hinic_clean_rq - Clean HW Receive Queue's Resources * @rq: HW Receive Queue **/ void hinic_clean_rq(struct hinic_rq *rq) { … } /** * hinic_get_sq_free_wqebbs - return number of free wqebbs for use * @sq: send queue * * Return number of free wqebbs **/ int hinic_get_sq_free_wqebbs(struct hinic_sq *sq) { … } /** * hinic_get_rq_free_wqebbs - return number of free wqebbs for use * @rq: recv queue * * Return number of free wqebbs **/ int hinic_get_rq_free_wqebbs(struct hinic_rq *rq) { … } static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs) { … } static void sq_prepare_task(struct hinic_sq_task *task) { … } void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len) { … } void hinic_task_set_outter_l3(struct hinic_sq_task *task, enum hinic_l3_offload_type l3_type, u32 network_len) { … } void hinic_task_set_inner_l3(struct hinic_sq_task *task, enum hinic_l3_offload_type l3_type, u32 network_len) { … } void hinic_task_set_tunnel_l4(struct hinic_sq_task *task, enum hinic_l4_tunnel_type l4_type, u32 tunnel_len) { … } void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info, enum hinic_l4_offload_type l4_offload, u32 l4_len, u32 offset) { … } void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info, enum hinic_l4_offload_type l4_offload, u32 l4_len, u32 offset, u32 ip_ident, u32 mss) { … } /** * hinic_sq_prepare_wqe - prepare wqe before insert to the queue * @sq: send queue * @sq_wqe: wqe to prepare * @sges: sges for use by the wqe for send for buf addresses * @nr_sges: number of sges **/ void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges, int nr_sges) { … } /** * sq_prepare_db - prepare doorbell to write * @sq: send queue * @prod_idx: pi value for the doorbell * @cos: cos of the doorbell * * Return db value **/ static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos) { … } /** * hinic_sq_write_db- write doorbell * @sq: send queue * @prod_idx: pi value for the doorbell * @wqe_size: wqe size * @cos: cos of the wqe **/ void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size, unsigned int cos) { … } /** * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi * @sq: sq to get wqe from * @wqe_size: wqe size * @prod_idx: returned pi * * Return wqe pointer **/ struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq, unsigned int wqe_size, u16 *prod_idx) { … } /** * hinic_sq_return_wqe - return the wqe to the sq * @sq: send queue * @wqe_size: the size of the wqe **/ void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size) { … } /** * hinic_sq_write_wqe - write the wqe to the sq * @sq: send queue * @prod_idx: pi of the wqe * @sq_wqe: the wqe to write * @skb: skb to save * @wqe_size: the size of the wqe **/ void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx, struct hinic_sq_wqe *sq_wqe, struct sk_buff *skb, unsigned int wqe_size) { … } /** * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the * wqe only have one wqebb * @sq: send queue * @skb: return skb that was saved * @wqe_size: the wqe size ptr * @cons_idx: consumer index of the wqe * * Return wqe in ci position **/ struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq, struct sk_buff **skb, unsigned int *wqe_size, u16 *cons_idx) { … } /** * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci * @sq: send queue * @skb: return skb that was saved * @wqe_size: the size of the wqe * @cons_idx: consumer index of the wqe * * Return wqe in ci position **/ struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq, struct sk_buff **skb, unsigned int wqe_size, u16 *cons_idx) { … } /** * hinic_sq_put_wqe - release the ci for new wqes * @sq: send queue * @wqe_size: the size of the wqe **/ void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size) { … } /** * hinic_sq_get_sges - get sges from the wqe * @sq_wqe: wqe to get the sges from its buffer addresses * @sges: returned sges * @nr_sges: number sges to return **/ void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges, int nr_sges) { … } /** * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi * @rq: rq to get wqe from * @wqe_size: wqe size * @prod_idx: returned pi * * Return wqe pointer **/ struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq, unsigned int wqe_size, u16 *prod_idx) { … } /** * hinic_rq_write_wqe - write the wqe to the rq * @rq: recv queue * @prod_idx: pi of the wqe * @rq_wqe: the wqe to write * @skb: skb to save **/ void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx, struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb) { … } /** * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci * @rq: recv queue * @wqe_size: the size of the wqe * @skb: return saved skb * @cons_idx: consumer index of the wqe * * Return wqe in ci position **/ struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq, unsigned int wqe_size, struct sk_buff **skb, u16 *cons_idx) { … } /** * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position * @rq: recv queue * @wqe_size: the size of the wqe * @skb: return saved skb * @cons_idx: consumer index in the wq * * Return wqe in incremented ci position **/ struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq, unsigned int wqe_size, struct sk_buff **skb, u16 *cons_idx) { … } /** * hinic_rq_put_wqe - release the ci for new wqes * @rq: recv queue * @cons_idx: consumer index of the wqe * @wqe_size: the size of the wqe **/ void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx, unsigned int wqe_size) { … } /** * hinic_rq_get_sge - get sge from the wqe * @rq: recv queue * @rq_wqe: wqe to get the sge from its buf address * @cons_idx: consumer index * @sge: returned sge **/ void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe, u16 cons_idx, struct hinic_sge *sge) { … } /** * hinic_rq_prepare_wqe - prepare wqe before insert to the queue * @rq: recv queue * @prod_idx: pi value * @rq_wqe: the wqe * @sge: sge for use by the wqe for recv buf address **/ void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx, struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge) { … } /** * hinic_rq_update - update pi of the rq * @rq: recv queue * @prod_idx: pi value **/ void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx) { … }
Codebase Browser
linux