// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Cavium, Inc. */ #include <linux/pci.h> #include <linux/netdevice.h> #include <linux/ip.h> #include <linux/etherdevice.h> #include <linux/iommu.h> #include <net/ip.h> #include <net/tso.h> #include <uapi/linux/bpf.h> #include "nic_reg.h" #include "nic.h" #include "q_struct.h" #include "nicvf_queues.h" static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, int size, u64 data); static void nicvf_get_page(struct nicvf *nic) { … } /* Poll a register for a specific value */ static int nicvf_poll_reg(struct nicvf *nic, int qidx, u64 reg, int bit_pos, int bits, int val) { … } /* Allocate memory for a queue's descriptors */ static int nicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem, int q_len, int desc_size, int align_bytes) { … } /* Free queue's descriptor memory */ static void nicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem) { … } #define XDP_PAGE_REFCNT_REFILL … /* Allocate a new page or recycle one if possible * * We cannot optimize dma mapping here, since * 1. It's only one RBDR ring for 8 Rx queues. * 2. CQE_RX gives address of the buffer where pkt has been DMA'ed * and not idx into RBDR ring, so can't refer to saved info. * 3. There are multiple receive buffers per page */ static inline struct pgcache *nicvf_alloc_page(struct nicvf *nic, struct rbdr *rbdr, gfp_t gfp) { … } /* Allocate buffer for packet reception */ static inline int nicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr, gfp_t gfp, u32 buf_len, u64 *rbuf) { … } /* Build skb around receive buffer */ static struct sk_buff *nicvf_rb_ptr_to_skb(struct nicvf *nic, u64 rb_ptr, int len) { … } /* Allocate RBDR ring and populate receive buffers */ static int nicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, int ring_len, int buf_size) { … } /* Free RBDR ring and its receive buffers */ static void nicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr) { … } /* Refill receive buffer descriptors with new buffers. */ static void nicvf_refill_rbdr(struct nicvf *nic, gfp_t gfp) { … } /* Alloc rcv buffers in non-atomic mode for better success */ void nicvf_rbdr_work(struct work_struct *work) { … } /* In Softirq context, alloc rcv buffers in atomic mode */ void nicvf_rbdr_task(struct tasklet_struct *t) { … } /* Initialize completion queue */ static int nicvf_init_cmp_queue(struct nicvf *nic, struct cmp_queue *cq, int q_len) { … } static void nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq) { … } /* Initialize transmit queue */ static int nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len, int qidx) { … } void nicvf_unmap_sndq_buffers(struct nicvf *nic, struct snd_queue *sq, int hdr_sqe, u8 subdesc_cnt) { … } static void nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) { … } static void nicvf_reclaim_snd_queue(struct nicvf *nic, struct queue_set *qs, int qidx) { … } static void nicvf_reclaim_rcv_queue(struct nicvf *nic, struct queue_set *qs, int qidx) { … } static void nicvf_reclaim_cmp_queue(struct nicvf *nic, struct queue_set *qs, int qidx) { … } static void nicvf_reclaim_rbdr(struct nicvf *nic, struct rbdr *rbdr, int qidx) { … } void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features) { … } static void nicvf_reset_rcv_queue_stats(struct nicvf *nic) { … } /* Configures receive queue */ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx, bool enable) { … } /* Configures completion queue */ void nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx, bool enable) { … } /* Configures transmit queue */ static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx, bool enable) { … } /* Configures receive buffer descriptor ring */ static void nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, int qidx, bool enable) { … } /* Requests PF to assign and enable Qset */ void nicvf_qset_config(struct nicvf *nic, bool enable) { … } static void nicvf_free_resources(struct nicvf *nic) { … } static int nicvf_alloc_resources(struct nicvf *nic) { … } int nicvf_set_qset_resources(struct nicvf *nic) { … } int nicvf_config_data_transfer(struct nicvf *nic, bool enable) { … } /* Get a free desc from SQ * returns descriptor ponter & descriptor number */ static inline int nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt) { … } /* Rollback to previous tail pointer when descriptors not used */ static inline void nicvf_rollback_sq_desc(struct snd_queue *sq, int qentry, int desc_cnt) { … } /* Free descriptor back to SQ for future use */ void nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt) { … } static inline int nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry) { … } void nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx) { … } void nicvf_sq_disable(struct nicvf *nic, int qidx) { … } void nicvf_sq_free_used_descs(struct net_device *netdev, struct snd_queue *sq, int qidx) { … } /* XDP Transmit APIs */ void nicvf_xdp_sq_doorbell(struct nicvf *nic, struct snd_queue *sq, int sq_num) { … } static inline void nicvf_xdp_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry, int subdesc_cnt, u64 data, int len) { … } int nicvf_xdp_sq_append_pkt(struct nicvf *nic, struct snd_queue *sq, u64 bufaddr, u64 dma_addr, u16 len) { … } /* Calculate no of SQ subdescriptors needed to transmit all * segments of this TSO packet. * Taken from 'Tilera network driver' with a minor modification. */ static int nicvf_tso_count_subdescs(struct sk_buff *skb) { … } #define POST_CQE_DESC_COUNT … /* Get the number of SQ descriptors needed to xmit this skb */ static int nicvf_sq_subdesc_required(struct nicvf *nic, struct sk_buff *skb) { … } /* Add SQ HEADER subdescriptor. * First subdescriptor for every send descriptor. */ static inline void nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry, int subdesc_cnt, struct sk_buff *skb, int len) { … } /* SQ GATHER subdescriptor * Must follow HDR descriptor */ static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, int size, u64 data) { … } /* Add HDR + IMMEDIATE subdescriptors right after descriptors of a TSO * packet so that a CQE is posted as a notifation for transmission of * TSO packet. */ static inline void nicvf_sq_add_cqe_subdesc(struct snd_queue *sq, int qentry, int tso_sqe, struct sk_buff *skb) { … } static inline void nicvf_sq_doorbell(struct nicvf *nic, struct sk_buff *skb, int sq_num, int desc_cnt) { … } /* Segment a TSO packet into 'gso_size' segments and append * them to SQ for transfer */ static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq, int sq_num, int qentry, struct sk_buff *skb) { … } /* Append an skb to a SQ for packet transfer. */ int nicvf_sq_append_skb(struct nicvf *nic, struct snd_queue *sq, struct sk_buff *skb, u8 sq_num) { … } static inline unsigned frag_num(unsigned i) { … } static void nicvf_unmap_rcv_buffer(struct nicvf *nic, u64 dma_addr, u64 buf_addr, bool xdp) { … } /* Returns SKB for a received packet */ struct sk_buff *nicvf_get_rcv_skb(struct nicvf *nic, struct cqe_rx_t *cqe_rx, bool xdp) { … } static u64 nicvf_int_type_to_mask(int int_type, int q_idx) { … } /* Enable interrupt */ void nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx) { … } /* Disable interrupt */ void nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx) { … } /* Clear interrupt */ void nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx) { … } /* Check if interrupt is enabled */ int nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx) { … } void nicvf_update_rq_stats(struct nicvf *nic, int rq_idx) { … } void nicvf_update_sq_stats(struct nicvf *nic, int sq_idx) { … } /* Check for errors in the receive cmp.queue entry */ int nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cqe_rx_t *cqe_rx) { … } /* Check for errors in the send cmp.queue entry */ int nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cqe_send_t *cqe_tx) { … }
Codebase Browser
linux