// SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2018 Intel Corporation. */ #include <linux/bpf_trace.h> #include <net/xdp_sock_drv.h> #include "i40e_txrx_common.h" #include "i40e_xsk.h" void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring) { … } static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx) { … } /** * i40e_realloc_rx_xdp_bi - reallocate SW ring for either XSK or normal buffer * @rx_ring: Current rx ring * @pool_present: is pool for XSK present * * Try allocating memory and return ENOMEM, if failed to allocate. * If allocation was successful, substitute buffer with allocated one. * Returns 0 on success, negative on failure */ static int i40e_realloc_rx_xdp_bi(struct i40e_ring *rx_ring, bool pool_present) { … } /** * i40e_realloc_rx_bi_zc - reallocate rx SW rings * @vsi: Current VSI * @zc: is zero copy set * * Reallocate buffer for rx_rings that might be used by XSK. * XDP requires more memory, than rx_buf provides. * Returns 0 on success, negative on failure */ int i40e_realloc_rx_bi_zc(struct i40e_vsi *vsi, bool zc) { … } /** * i40e_xsk_pool_enable - Enable/associate an AF_XDP buffer pool to a * certain ring/qid * @vsi: Current VSI * @pool: buffer pool * @qid: Rx ring to associate buffer pool with * * Returns 0 on success, <0 on failure **/ static int i40e_xsk_pool_enable(struct i40e_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) { … } /** * i40e_xsk_pool_disable - Disassociate an AF_XDP buffer pool from a * certain ring/qid * @vsi: Current VSI * @qid: Rx ring to associate buffer pool with * * Returns 0 on success, <0 on failure **/ static int i40e_xsk_pool_disable(struct i40e_vsi *vsi, u16 qid) { … } /** * i40e_xsk_pool_setup - Enable/disassociate an AF_XDP buffer pool to/from * a ring/qid * @vsi: Current VSI * @pool: Buffer pool to enable/associate to a ring, or NULL to disable * @qid: Rx ring to (dis)associate buffer pool (from)to * * This function enables or disables a buffer pool to a certain ring. * * Returns 0 on success, <0 on failure **/ int i40e_xsk_pool_setup(struct i40e_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) { … } /** * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff * @rx_ring: Rx ring * @xdp: xdp_buff used as input to the XDP program * @xdp_prog: XDP program to run * * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR} **/ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { … } bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count) { … } /** * i40e_construct_skb_zc - Create skbuff from zero-copy Rx buffer * @rx_ring: Rx ring * @xdp: xdp_buff * * This functions allocates a new skb from a zero-copy Rx buffer. * * Returns the skb, or NULL on failure. **/ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) { … } static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp_buff, union i40e_rx_desc *rx_desc, unsigned int *rx_packets, unsigned int *rx_bytes, unsigned int xdp_res, bool *failure) { … } static int i40e_add_xsk_frag(struct i40e_ring *rx_ring, struct xdp_buff *first, struct xdp_buff *xdp, const unsigned int size) { … } /** * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring * @rx_ring: Rx ring * @budget: NAPI budget * * Returns amount of work completed **/ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) { … } static void i40e_xmit_pkt(struct i40e_ring *xdp_ring, struct xdp_desc *desc, unsigned int *total_bytes) { … } static void i40e_xmit_pkt_batch(struct i40e_ring *xdp_ring, struct xdp_desc *desc, unsigned int *total_bytes) { … } static void i40e_fill_tx_hw_ring(struct i40e_ring *xdp_ring, struct xdp_desc *descs, u32 nb_pkts, unsigned int *total_bytes) { … } static void i40e_set_rs_bit(struct i40e_ring *xdp_ring) { … } /** * i40e_xmit_zc - Performs zero-copy Tx AF_XDP * @xdp_ring: XDP Tx ring * @budget: NAPI budget * * Returns true if the work is finished. **/ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) { … } /** * i40e_clean_xdp_tx_buffer - Frees and unmaps an XDP Tx entry * @tx_ring: XDP Tx ring * @tx_bi: Tx buffer info to clean **/ static void i40e_clean_xdp_tx_buffer(struct i40e_ring *tx_ring, struct i40e_tx_buffer *tx_bi) { … } /** * i40e_clean_xdp_tx_irq - Completes AF_XDP entries, and cleans XDP entries * @vsi: Current VSI * @tx_ring: XDP Tx ring * * Returns true if cleanup/transmission is done. **/ bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi, struct i40e_ring *tx_ring) { … } /** * i40e_xsk_wakeup - Implements the ndo_xsk_wakeup * @dev: the netdevice * @queue_id: queue id to wake up * @flags: ignored in our case since we have Rx and Tx in the same NAPI. * * Returns <0 for errors, 0 otherwise. **/ int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) { … } void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring) { … } /** * i40e_xsk_clean_tx_ring - Clean the XDP Tx ring on shutdown * @tx_ring: XDP Tx ring **/ void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring) { … } /** * i40e_xsk_any_rx_ring_enabled - Checks if Rx rings have an AF_XDP * buffer pool attached * @vsi: vsi * * Returns true if any of the Rx rings has an AF_XDP buffer pool attached **/ bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi) { … }
Codebase Browser
linux