// SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ #include <net/libeth/rx.h> #include <net/libeth/tx.h> #include "idpf.h" #include "idpf_virtchnl.h" struct idpf_tx_stash { … }; #define idpf_tx_buf_compl_tag(buf) … LIBETH_SQE_CHECK_PRIV(…) …; static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, unsigned int count); /** * idpf_buf_lifo_push - push a buffer pointer onto stack * @stack: pointer to stack struct * @buf: pointer to buf to push * * Returns 0 on success, negative on failure **/ static int idpf_buf_lifo_push(struct idpf_buf_lifo *stack, struct idpf_tx_stash *buf) { … } /** * idpf_buf_lifo_pop - pop a buffer pointer from stack * @stack: pointer to stack struct **/ static struct idpf_tx_stash *idpf_buf_lifo_pop(struct idpf_buf_lifo *stack) { … } /** * idpf_tx_timeout - Respond to a Tx Hang * @netdev: network interface device structure * @txqueue: TX queue */ void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) { … } /** * idpf_tx_buf_rel_all - Free any empty Tx buffers * @txq: queue to be cleaned */ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) { … } /** * idpf_tx_desc_rel - Free Tx resources per queue * @txq: Tx descriptor ring for a specific queue * * Free all transmit software resources */ static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) { … } /** * idpf_compl_desc_rel - Free completion resources per queue * @complq: completion queue * * Free all completion software resources. */ static void idpf_compl_desc_rel(struct idpf_compl_queue *complq) { … } /** * idpf_tx_desc_rel_all - Free Tx Resources for All Queues * @vport: virtual port structure * * Free all transmit software resources */ static void idpf_tx_desc_rel_all(struct idpf_vport *vport) { … } /** * idpf_tx_buf_alloc_all - Allocate memory for all buffer resources * @tx_q: queue for which the buffers are allocated * * Returns 0 on success, negative on failure */ static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q) { … } /** * idpf_tx_desc_alloc - Allocate the Tx descriptors * @vport: vport to allocate resources for * @tx_q: the tx ring to set up * * Returns 0 on success, negative on failure */ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, struct idpf_tx_queue *tx_q) { … } /** * idpf_compl_desc_alloc - allocate completion descriptors * @vport: vport to allocate resources for * @complq: completion queue to set up * * Return: 0 on success, -errno on failure. */ static int idpf_compl_desc_alloc(const struct idpf_vport *vport, struct idpf_compl_queue *complq) { … } /** * idpf_tx_desc_alloc_all - allocate all queues Tx resources * @vport: virtual port private structure * * Returns 0 on success, negative on failure */ static int idpf_tx_desc_alloc_all(struct idpf_vport *vport) { … } /** * idpf_rx_page_rel - Release an rx buffer page * @rx_buf: the buffer to free */ static void idpf_rx_page_rel(struct libeth_fqe *rx_buf) { … } /** * idpf_rx_hdr_buf_rel_all - Release header buffer memory * @bufq: queue to use */ static void idpf_rx_hdr_buf_rel_all(struct idpf_buf_queue *bufq) { … } /** * idpf_rx_buf_rel_bufq - Free all Rx buffer resources for a buffer queue * @bufq: queue to be cleaned */ static void idpf_rx_buf_rel_bufq(struct idpf_buf_queue *bufq) { … } /** * idpf_rx_buf_rel_all - Free all Rx buffer resources for a receive queue * @rxq: queue to be cleaned */ static void idpf_rx_buf_rel_all(struct idpf_rx_queue *rxq) { … } /** * idpf_rx_desc_rel - Free a specific Rx q resources * @rxq: queue to clean the resources from * @dev: device to free DMA memory * @model: single or split queue model * * Free a specific rx queue resources */ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, u32 model) { … } /** * idpf_rx_desc_rel_bufq - free buffer queue resources * @bufq: buffer queue to clean the resources from * @dev: device to free DMA memory */ static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq, struct device *dev) { … } /** * idpf_rx_desc_rel_all - Free Rx Resources for All Queues * @vport: virtual port structure * * Free all rx queues resources */ static void idpf_rx_desc_rel_all(struct idpf_vport *vport) { … } /** * idpf_rx_buf_hw_update - Store the new tail and head values * @bufq: queue to bump * @val: new head index */ static void idpf_rx_buf_hw_update(struct idpf_buf_queue *bufq, u32 val) { … } /** * idpf_rx_hdr_buf_alloc_all - Allocate memory for header buffers * @bufq: ring to use * * Returns 0 on success, negative on failure. */ static int idpf_rx_hdr_buf_alloc_all(struct idpf_buf_queue *bufq) { … } /** * idpf_rx_post_buf_refill - Post buffer id to refill queue * @refillq: refill queue to post to * @buf_id: buffer id to post */ static void idpf_rx_post_buf_refill(struct idpf_sw_queue *refillq, u16 buf_id) { … } /** * idpf_rx_post_buf_desc - Post buffer to bufq descriptor ring * @bufq: buffer queue to post to * @buf_id: buffer id to post * * Returns false if buffer could not be allocated, true otherwise. */ static bool idpf_rx_post_buf_desc(struct idpf_buf_queue *bufq, u16 buf_id) { … } /** * idpf_rx_post_init_bufs - Post initial buffers to bufq * @bufq: buffer queue to post working set to * @working_set: number of buffers to put in working set * * Returns true if @working_set bufs were posted successfully, false otherwise. */ static bool idpf_rx_post_init_bufs(struct idpf_buf_queue *bufq, u16 working_set) { … } /** * idpf_rx_buf_alloc_singleq - Allocate memory for all buffer resources * @rxq: queue for which the buffers are allocated * * Return: 0 on success, -ENOMEM on failure. */ static int idpf_rx_buf_alloc_singleq(struct idpf_rx_queue *rxq) { … } /** * idpf_rx_bufs_init_singleq - Initialize page pool and allocate Rx bufs * @rxq: buffer queue to create page pool for * * Return: 0 on success, -errno on failure. */ static int idpf_rx_bufs_init_singleq(struct idpf_rx_queue *rxq) { … } /** * idpf_rx_buf_alloc_all - Allocate memory for all buffer resources * @rxbufq: queue for which the buffers are allocated * * Returns 0 on success, negative on failure */ static int idpf_rx_buf_alloc_all(struct idpf_buf_queue *rxbufq) { … } /** * idpf_rx_bufs_init - Initialize page pool, allocate rx bufs, and post to HW * @bufq: buffer queue to create page pool for * @type: type of Rx buffers to allocate * * Returns 0 on success, negative on failure */ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, enum libeth_fqe_type type) { … } /** * idpf_rx_bufs_init_all - Initialize all RX bufs * @vport: virtual port struct * * Returns 0 on success, negative on failure */ int idpf_rx_bufs_init_all(struct idpf_vport *vport) { … } /** * idpf_rx_desc_alloc - Allocate queue Rx resources * @vport: vport to allocate resources for * @rxq: Rx queue for which the resources are setup * * Returns 0 on success, negative on failure */ static int idpf_rx_desc_alloc(const struct idpf_vport *vport, struct idpf_rx_queue *rxq) { … } /** * idpf_bufq_desc_alloc - Allocate buffer queue descriptor ring * @vport: vport to allocate resources for * @bufq: buffer queue for which the resources are set up * * Return: 0 on success, -ENOMEM on failure. */ static int idpf_bufq_desc_alloc(const struct idpf_vport *vport, struct idpf_buf_queue *bufq) { … } /** * idpf_rx_desc_alloc_all - allocate all RX queues resources * @vport: virtual port structure * * Returns 0 on success, negative on failure */ static int idpf_rx_desc_alloc_all(struct idpf_vport *vport) { … } /** * idpf_txq_group_rel - Release all resources for txq groups * @vport: vport to release txq groups on */ static void idpf_txq_group_rel(struct idpf_vport *vport) { … } /** * idpf_rxq_sw_queue_rel - Release software queue resources * @rx_qgrp: rx queue group with software queues */ static void idpf_rxq_sw_queue_rel(struct idpf_rxq_group *rx_qgrp) { … } /** * idpf_rxq_group_rel - Release all resources for rxq groups * @vport: vport to release rxq groups on */ static void idpf_rxq_group_rel(struct idpf_vport *vport) { … } /** * idpf_vport_queue_grp_rel_all - Release all queue groups * @vport: vport to release queue groups for */ static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport) { … } /** * idpf_vport_queues_rel - Free memory for all queues * @vport: virtual port * * Free the memory allocated for queues associated to a vport */ void idpf_vport_queues_rel(struct idpf_vport *vport) { … } /** * idpf_vport_init_fast_path_txqs - Initialize fast path txq array * @vport: vport to init txqs on * * We get a queue index from skb->queue_mapping and we need a fast way to * dereference the queue from queue groups. This allows us to quickly pull a * txq based on a queue index. * * Returns 0 on success, negative on failure */ static int idpf_vport_init_fast_path_txqs(struct idpf_vport *vport) { … } /** * idpf_vport_init_num_qs - Initialize number of queues * @vport: vport to initialize queues * @vport_msg: data to be filled into vport */ void idpf_vport_init_num_qs(struct idpf_vport *vport, struct virtchnl2_create_vport *vport_msg) { … } /** * idpf_vport_calc_num_q_desc - Calculate number of queue groups * @vport: vport to calculate q groups for */ void idpf_vport_calc_num_q_desc(struct idpf_vport *vport) { … } /** * idpf_vport_calc_total_qs - Calculate total number of queues * @adapter: private data struct * @vport_idx: vport idx to retrieve vport pointer * @vport_msg: message to fill with data * @max_q: vport max queue info * * Return 0 on success, error value on failure. */ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, struct virtchnl2_create_vport *vport_msg, struct idpf_vport_max_q *max_q) { … } /** * idpf_vport_calc_num_q_groups - Calculate number of queue groups * @vport: vport to calculate q groups for */ void idpf_vport_calc_num_q_groups(struct idpf_vport *vport) { … } /** * idpf_vport_calc_numq_per_grp - Calculate number of queues per group * @vport: vport to calculate queues for * @num_txq: return parameter for number of TX queues * @num_rxq: return parameter for number of RX queues */ static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport, u16 *num_txq, u16 *num_rxq) { … } /** * idpf_rxq_set_descids - set the descids supported by this queue * @vport: virtual port data structure * @q: rx queue for which descids are set * */ static void idpf_rxq_set_descids(const struct idpf_vport *vport, struct idpf_rx_queue *q) { … } /** * idpf_txq_group_alloc - Allocate all txq group resources * @vport: vport to allocate txq groups for * @num_txq: number of txqs to allocate for each group * * Returns 0 on success, negative on failure */ static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) { … } /** * idpf_rxq_group_alloc - Allocate all rxq group resources * @vport: vport to allocate rxq groups for * @num_rxq: number of rxqs to allocate for each group * * Returns 0 on success, negative on failure */ static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq) { … } /** * idpf_vport_queue_grp_alloc_all - Allocate all queue groups/resources * @vport: vport with qgrps to allocate * * Returns 0 on success, negative on failure */ static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport) { … } /** * idpf_vport_queues_alloc - Allocate memory for all queues * @vport: virtual port * * Allocate memory for queues associated with a vport. Returns 0 on success, * negative on failure. */ int idpf_vport_queues_alloc(struct idpf_vport *vport) { … } /** * idpf_tx_handle_sw_marker - Handle queue marker packet * @tx_q: tx queue to handle software marker */ static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) { … } /** * idpf_tx_clean_stashed_bufs - clean bufs that were stored for * out of order completions * @txq: queue to clean * @compl_tag: completion tag of packet to clean (from completion descriptor) * @cleaned: pointer to stats struct to track cleaned packets/bytes * @budget: Used to determine if we are in netpoll */ static void idpf_tx_clean_stashed_bufs(struct idpf_tx_queue *txq, u16 compl_tag, struct libeth_sq_napi_stats *cleaned, int budget) { … } /** * idpf_stash_flow_sch_buffers - store buffer parameters info to be freed at a * later time (only relevant for flow scheduling mode) * @txq: Tx queue to clean * @tx_buf: buffer to store */ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, struct idpf_tx_buf *tx_buf) { … } #define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) … /** * idpf_tx_splitq_clean - Reclaim resources from buffer queue * @tx_q: Tx queue to clean * @end: queue index until which it should be cleaned * @napi_budget: Used to determine if we are in netpoll * @cleaned: pointer to stats struct to track cleaned packets/bytes * @descs_only: true if queue is using flow-based scheduling and should * not clean buffers at this time * * Cleans the queue descriptor ring. If the queue is using queue-based * scheduling, the buffers will be cleaned as well. If the queue is using * flow-based scheduling, only the descriptors are cleaned at this time. * Separate packet completion events will be reported on the completion queue, * and the buffers will be cleaned separately. The stats are not updated from * this function when using flow-based scheduling. * * Furthermore, in flow scheduling mode, check to make sure there are enough * reserve buffers to stash the packet. If there are not, return early, which * will leave next_to_clean pointing to the packet that failed to be stashed. * * Return: false in the scenario above, true otherwise. */ static bool idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, int napi_budget, struct libeth_sq_napi_stats *cleaned, bool descs_only) { … } #define idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, buf) … /** * idpf_tx_clean_buf_ring - clean flow scheduling TX queue buffers * @txq: queue to clean * @compl_tag: completion tag of packet to clean (from completion descriptor) * @cleaned: pointer to stats struct to track cleaned packets/bytes * @budget: Used to determine if we are in netpoll * * Cleans all buffers associated with the input completion tag either from the * TX buffer ring or from the hash table if the buffers were previously * stashed. Returns the byte/segment count for the cleaned packet associated * this completion tag. */ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, struct libeth_sq_napi_stats *cleaned, int budget) { … } /** * idpf_tx_handle_rs_completion - clean a single packet and all of its buffers * whether on the buffer ring or in the hash table * @txq: Tx ring to clean * @desc: pointer to completion queue descriptor to extract completion * information from * @cleaned: pointer to stats struct to track cleaned packets/bytes * @budget: Used to determine if we are in netpoll * * Returns bytes/packets cleaned */ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, struct idpf_splitq_tx_compl_desc *desc, struct libeth_sq_napi_stats *cleaned, int budget) { … } /** * idpf_tx_clean_complq - Reclaim resources on completion queue * @complq: Tx ring to clean * @budget: Used to determine if we are in netpoll * @cleaned: returns number of packets cleaned * * Returns true if there's any budget left (e.g. the clean is finished) */ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, int *cleaned) { … } /** * idpf_tx_splitq_build_ctb - populate command tag and size for queue * based scheduling descriptors * @desc: descriptor to populate * @params: pointer to tx params struct * @td_cmd: command to be filled in desc * @size: size of buffer */ void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { … } /** * idpf_tx_splitq_build_flow_desc - populate command tag and size for flow * scheduling descriptors * @desc: descriptor to populate * @params: pointer to tx params struct * @td_cmd: command to be filled in desc * @size: size of buffer */ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { … } /** * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions * @tx_q: the queue to be checked * @descs_needed: number of descriptors required for this packet * * Returns 0 if stop is not needed */ static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, unsigned int descs_needed) { … } /** * idpf_tx_buf_hw_update - Store the new tail value * @tx_q: queue to bump * @val: new tail index * @xmit_more: more skb's pending * * The naming here is special in that 'hw' signals that this function is about * to do a register write to update our queue status. We know this can only * mean tail here as HW should be owning head for TX. */ void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, bool xmit_more) { … } /** * idpf_tx_desc_count_required - calculate number of Tx descriptors needed * @txq: queue to send buffer on * @skb: send buffer * * Returns number of data descriptors needed for this skb. */ unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, struct sk_buff *skb) { … } /** * idpf_tx_dma_map_error - handle TX DMA map errors * @txq: queue to send buffer on * @skb: send buffer * @first: original first buffer info buffer for packet * @idx: starting point on ring to unwind */ void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *first, u16 idx) { … } /** * idpf_tx_splitq_bump_ntu - adjust NTU and generation * @txq: the tx ring to wrap * @ntu: ring index to bump */ static unsigned int idpf_tx_splitq_bump_ntu(struct idpf_tx_queue *txq, u16 ntu) { … } /** * idpf_tx_splitq_map - Build the Tx flex descriptor * @tx_q: queue to send buffer on * @params: pointer to splitq params struct * @first: first buffer info buffer to use * * This function loops over the skb data pointed to by *first * and gets a physical address for each memory location and programs * it and the length into the transmit flex descriptor. */ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, struct idpf_tx_splitq_params *params, struct idpf_tx_buf *first) { … } /** * idpf_tso - computes mss and TSO length to prepare for TSO * @skb: pointer to skb * @off: pointer to struct that holds offload parameters * * Returns error (negative) if TSO was requested but cannot be applied to the * given skb, 0 if TSO does not apply to the given skb, or 1 otherwise. */ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) { … } /** * __idpf_chk_linearize - Check skb is not using too many buffers * @skb: send buffer * @max_bufs: maximum number of buffers * * For TSO we need to count the TSO header and segment payload separately. As * such we need to check cases where we have max_bufs-1 fragments or more as we * can potentially require max_bufs+1 DMA transactions, 1 for the TSO header, 1 * for the segment payload in the first descriptor, and another max_buf-1 for * the fragments. */ static bool __idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs) { … } /** * idpf_chk_linearize - Check if skb exceeds max descriptors per packet * @skb: send buffer * @max_bufs: maximum scatter gather buffers for single packet * @count: number of buffers this packet needs * * Make sure we don't exceed maximum scatter gather buffers for a single * packet. We have to do some special checking around the boundary (max_bufs-1) * if TSO is on since we need count the TSO header and payload separately. * E.g.: a packet with 7 fragments can require 9 DMA transactions; 1 for TSO * header, 1 for segment payload, and then 7 for the fragments. */ static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, unsigned int count) { … } /** * idpf_tx_splitq_get_ctx_desc - grab next desc and update buffer ring * @txq: queue to put context descriptor on * * Since the TX buffer rings mimics the descriptor ring, update the tx buffer * ring entry to reflect that this index is a context descriptor */ static struct idpf_flex_tx_ctx_desc * idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) { … } /** * idpf_tx_drop_skb - free the SKB and bump tail if necessary * @tx_q: queue to send buffer on * @skb: pointer to skb */ netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb) { … } /** * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors * @skb: send buffer * @tx_q: queue to send buffer on * * Returns NETDEV_TX_OK if sent, else an error code */ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, struct idpf_tx_queue *tx_q) { … } /** * idpf_tx_start - Selects the right Tx queue to send buffer * @skb: send buffer * @netdev: network interface device structure * * Returns NETDEV_TX_OK if sent, else an error code */ netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev) { … } /** * idpf_rx_hash - set the hash value in the skb * @rxq: Rx descriptor ring packet is being transacted on * @skb: pointer to current skb being populated * @rx_desc: Receive descriptor * @decoded: Decoded Rx packet type related fields */ static void idpf_rx_hash(const struct idpf_rx_queue *rxq, struct sk_buff *skb, const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, struct libeth_rx_pt decoded) { … } /** * idpf_rx_csum - Indicate in skb if checksum is good * @rxq: Rx descriptor ring packet is being transacted on * @skb: pointer to current skb being populated * @csum_bits: checksum fields extracted from the descriptor * @decoded: Decoded Rx packet type related fields * * skb->protocol must be set before this function is called */ static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, struct idpf_rx_csum_decoded csum_bits, struct libeth_rx_pt decoded) { … } /** * idpf_rx_splitq_extract_csum_bits - Extract checksum bits from descriptor * @rx_desc: receive descriptor * * Return: parsed checksum status. **/ static struct idpf_rx_csum_decoded idpf_rx_splitq_extract_csum_bits(const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) { … } /** * idpf_rx_rsc - Set the RSC fields in the skb * @rxq : Rx descriptor ring packet is being transacted on * @skb : pointer to current skb being populated * @rx_desc: Receive descriptor * @decoded: Decoded Rx packet type related fields * * Return 0 on success and error code on failure * * Populate the skb fields with the total number of RSC segments, RSC payload * length and packet type. */ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, struct libeth_rx_pt decoded) { … } /** * idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor * @rxq: Rx descriptor ring packet is being transacted on * @skb: pointer to current skb being populated * @rx_desc: Receive descriptor * * This function checks the ring, descriptor, and packet information in * order to populate the hash, checksum, protocol, and * other fields within the skb. */ static int idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) { … } /** * idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag * @rx_buf: buffer containing page to add * @skb: sk_buff to place the data into * @size: packet length from rx_desc * * This function will add the data contained in rx_buf->page to the skb. * It will just attach the page as a frag to the skb. * The function will then update the page offset. */ void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, unsigned int size) { … } /** * idpf_rx_hsplit_wa - handle header buffer overflows and split errors * @hdr: Rx buffer for the headers * @buf: Rx buffer for the payload * @data_len: number of bytes received to the payload buffer * * When a header buffer overflow occurs or the HW was unable do parse the * packet type to perform header split, the whole frame gets placed to the * payload buffer. We can't build a valid skb around a payload buffer when * the header split is active since it doesn't reserve any head- or tailroom. * In that case, copy either the whole frame when it's short or just the * Ethernet header to the header buffer to be able to build an skb and adjust * the data offset in the payload buffer, IOW emulate the header split. * * Return: number of bytes copied to the header buffer. */ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr, struct libeth_fqe *buf, u32 data_len) { … } /** * idpf_rx_build_skb - Allocate skb and populate it from header buffer * @buf: Rx buffer to pull data from * @size: the length of the packet * * This function allocates an skb. It then populates it with the page data from * the current receive descriptor, taking care to set up the skb correctly. */ struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size) { … } /** * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor * status and error fields * @stat_err_field: field from descriptor to test bits in * @stat_err_bits: value to mask * */ static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field, const u8 stat_err_bits) { … } /** * idpf_rx_splitq_is_eop - process handling of EOP buffers * @rx_desc: Rx descriptor for current buffer * * If the buffer is an EOP buffer, this function exits returning true, * otherwise return false indicating that this is in fact a non-EOP buffer. */ static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) { … } /** * idpf_rx_splitq_clean - Clean completed descriptors from Rx queue * @rxq: Rx descriptor queue to retrieve receive buffer queue * @budget: Total limit on number of packets to process * * This function provides a "bounce buffer" approach to Rx interrupt * processing. The advantage to this is that on systems that have * expensive overhead for IOMMU access this provides a means of avoiding * it by maintaining the mapping of the page to the system. * * Returns amount of work completed */ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) { … } /** * idpf_rx_update_bufq_desc - Update buffer queue descriptor * @bufq: Pointer to the buffer queue * @buf_id: buffer ID * @buf_desc: Buffer queue descriptor * * Return 0 on success and negative on failure. */ static int idpf_rx_update_bufq_desc(struct idpf_buf_queue *bufq, u32 buf_id, struct virtchnl2_splitq_rx_buf_desc *buf_desc) { … } /** * idpf_rx_clean_refillq - Clean refill queue buffers * @bufq: buffer queue to post buffers back to * @refillq: refill queue to clean * * This function takes care of the buffer refill management */ static void idpf_rx_clean_refillq(struct idpf_buf_queue *bufq, struct idpf_sw_queue *refillq) { … } /** * idpf_rx_clean_refillq_all - Clean all refill queues * @bufq: buffer queue with refill queues * @nid: ID of the closest NUMA node with memory * * Iterates through all refill queues assigned to the buffer queue assigned to * this vector. Returns true if clean is complete within budget, false * otherwise. */ static void idpf_rx_clean_refillq_all(struct idpf_buf_queue *bufq, int nid) { … } /** * idpf_vport_intr_clean_queues - MSIX mode Interrupt Handler * @irq: interrupt number * @data: pointer to a q_vector * */ static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, void *data) { … } /** * idpf_vport_intr_napi_del_all - Unregister napi for all q_vectors in vport * @vport: virtual port structure * */ static void idpf_vport_intr_napi_del_all(struct idpf_vport *vport) { … } /** * idpf_vport_intr_napi_dis_all - Disable NAPI for all q_vectors in the vport * @vport: main vport structure */ static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport) { … } /** * idpf_vport_intr_rel - Free memory allocated for interrupt vectors * @vport: virtual port * * Free the memory allocated for interrupt vectors associated to a vport */ void idpf_vport_intr_rel(struct idpf_vport *vport) { … } /** * idpf_vport_intr_rel_irq - Free the IRQ association with the OS * @vport: main vport structure */ static void idpf_vport_intr_rel_irq(struct idpf_vport *vport) { … } /** * idpf_vport_intr_dis_irq_all - Disable all interrupt * @vport: main vport structure */ static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport) { … } /** * idpf_vport_intr_buildreg_itr - Enable default interrupt generation settings * @q_vector: pointer to q_vector * @type: itr index * @itr: itr value */ static u32 idpf_vport_intr_buildreg_itr(struct idpf_q_vector *q_vector, const int type, u16 itr) { … } /** * idpf_update_dim_sample - Update dim sample with packets and bytes * @q_vector: the vector associated with the interrupt * @dim_sample: dim sample to update * @dim: dim instance structure * @packets: total packets * @bytes: total bytes * * Update the dim sample with the packets and bytes which are passed to this * function. Set the dim state appropriately if the dim settings gets stale. */ static void idpf_update_dim_sample(struct idpf_q_vector *q_vector, struct dim_sample *dim_sample, struct dim *dim, u64 packets, u64 bytes) { … } /** * idpf_net_dim - Update net DIM algorithm * @q_vector: the vector associated with the interrupt * * Create a DIM sample and notify net_dim() so that it can possibly decide * a new ITR value based on incoming packets, bytes, and interrupts. * * This function is a no-op if the queue is not configured to dynamic ITR. */ static void idpf_net_dim(struct idpf_q_vector *q_vector) { … } /** * idpf_vport_intr_update_itr_ena_irq - Update itr and re-enable MSIX interrupt * @q_vector: q_vector for which itr is being updated and interrupt enabled * * Update the net_dim() algorithm and re-enable the interrupt associated with * this vector. */ void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector) { … } /** * idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport * @vport: main vport structure */ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) { … } /** * idpf_vport_intr_write_itr - Write ITR value to the ITR register * @q_vector: q_vector structure * @itr: Interrupt throttling rate * @tx: Tx or Rx ITR */ void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector, u16 itr, bool tx) { … } /** * idpf_vport_intr_ena_irq_all - Enable IRQ for the given vport * @vport: main vport structure */ static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport) { … } /** * idpf_vport_intr_deinit - Release all vector associations for the vport * @vport: main vport structure */ void idpf_vport_intr_deinit(struct idpf_vport *vport) { … } /** * idpf_tx_dim_work - Call back from the stack * @work: work queue structure */ static void idpf_tx_dim_work(struct work_struct *work) { … } /** * idpf_rx_dim_work - Call back from the stack * @work: work queue structure */ static void idpf_rx_dim_work(struct work_struct *work) { … } /** * idpf_init_dim - Set up dynamic interrupt moderation * @qv: q_vector structure */ static void idpf_init_dim(struct idpf_q_vector *qv) { … } /** * idpf_vport_intr_napi_ena_all - Enable NAPI for all q_vectors in the vport * @vport: main vport structure */ static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport) { … } /** * idpf_tx_splitq_clean_all- Clean completion queues * @q_vec: queue vector * @budget: Used to determine if we are in netpoll * @cleaned: returns number of packets cleaned * * Returns false if clean is not complete else returns true */ static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, int *cleaned) { … } /** * idpf_rx_splitq_clean_all- Clean completion queues * @q_vec: queue vector * @budget: Used to determine if we are in netpoll * @cleaned: returns number of packets cleaned * * Returns false if clean is not complete else returns true */ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, int *cleaned) { … } /** * idpf_vport_splitq_napi_poll - NAPI handler * @napi: struct from which you get q_vector * @budget: budget provided by stack */ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) { … } /** * idpf_vport_intr_map_vector_to_qs - Map vectors to queues * @vport: virtual port * * Mapping for vectors to queues */ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) { … } /** * idpf_vport_intr_init_vec_idx - Initialize the vector indexes * @vport: virtual port * * Initialize vector indexes with values returened over mailbox */ static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) { … } /** * idpf_vport_intr_napi_add_all- Register napi handler for all qvectors * @vport: virtual port structure */ static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport) { … } /** * idpf_vport_intr_alloc - Allocate memory for interrupt vectors * @vport: virtual port * * We allocate one q_vector per queue interrupt. If allocation fails we * return -ENOMEM. */ int idpf_vport_intr_alloc(struct idpf_vport *vport) { … } /** * idpf_vport_intr_init - Setup all vectors for the given vport * @vport: virtual port * * Returns 0 on success or negative on failure */ int idpf_vport_intr_init(struct idpf_vport *vport) { … } void idpf_vport_intr_ena(struct idpf_vport *vport) { … } /** * idpf_config_rss - Send virtchnl messages to configure RSS * @vport: virtual port * * Return 0 on success, negative on failure */ int idpf_config_rss(struct idpf_vport *vport) { … } /** * idpf_fill_dflt_rss_lut - Fill the indirection table with the default values * @vport: virtual port structure */ static void idpf_fill_dflt_rss_lut(struct idpf_vport *vport) { … } /** * idpf_init_rss - Allocate and initialize RSS resources * @vport: virtual port * * Return 0 on success, negative on failure */ int idpf_init_rss(struct idpf_vport *vport) { … } /** * idpf_deinit_rss - Release RSS resources * @vport: virtual port */ void idpf_deinit_rss(struct idpf_vport *vport) { … }
Codebase Browser
linux