/* * Copyright (c) 2007 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/mlx4/cq.h> #include <linux/slab.h> #include <linux/mlx4/qp.h> #include <linux/skbuff.h> #include <linux/rculist.h> #include <linux/if_ether.h> #include <linux/if_vlan.h> #include <linux/vmalloc.h> #include <linux/irq.h> #include <linux/skbuff_ref.h> #include <net/ip.h> #if IS_ENABLED(CONFIG_IPV6) #include <net/ip6_checksum.h> #endif #include "mlx4_en.h" static int mlx4_alloc_page(struct mlx4_en_priv *priv, struct mlx4_en_rx_alloc *frag, gfp_t gfp) { … } static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_alloc *frags, gfp_t gfp) { … } static void mlx4_en_free_frag(const struct mlx4_en_priv *priv, struct mlx4_en_rx_alloc *frag) { … } static void mlx4_en_init_rx_desc(const struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { … } static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index, gfp_t gfp) { … } static bool mlx4_en_is_ring_empty(const struct mlx4_en_rx_ring *ring) { … } static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) { … } /* slow path */ static void mlx4_en_free_rx_desc(const struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { … } /* Function not in fast-path */ static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) { … } static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { … } void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) { … } int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, u16 stride, int node, int queue_index) { … } int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { … } /* We recover from out of memory by scheduling our napi poll * function (mlx4_en_process_cq), which tries to allocate * all missing RX buffers (call to mlx4_en_refill_rx_buffers). */ void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv) { … } /* When the rx ring is running in page-per-packet mode, a released frame can go * directly into a small cache, to avoid unmapping or touching the page * allocator. In bpf prog performance scenarios, buffers are either forwarded * or dropped, never converted to skbs, so every page can come directly from * this cache when it is sized to be a multiple of the napi budget. */ bool mlx4_en_rx_recycle(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_alloc *frame) { … } void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, u16 stride) { … } void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { … } static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_alloc *frags, struct sk_buff *skb, int length) { … } static void validate_loopback(struct mlx4_en_priv *priv, void *va) { … } static void mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { … } /* When hardware doesn't strip the vlan, we need to calculate the checksum * over it and add it to the hardware's checksum calculation */ static inline __wsum get_fixed_vlan_csum(__wsum hw_checksum, struct vlan_hdr *vlanh) { … } /* Although the stack expects checksum which doesn't include the pseudo * header, the HW adds it. To address that, we are subtracting the pseudo * header checksum from the checksum value provided by the HW. */ static int get_fixed_ipv4_csum(__wsum hw_checksum, struct sk_buff *skb, struct iphdr *iph) { … } #if IS_ENABLED(CONFIG_IPV6) /* In IPv6 packets, hw_checksum lacks 6 bytes from IPv6 header: * 4 first bytes : priority, version, flow_lbl * and 2 additional bytes : nexthdr, hop_limit. */ static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb, struct ipv6hdr *ipv6h) { … } #endif #define short_frame(size) … /* We reach this function only after checking that any of * the (IPv4 | IPv6) bits are set in cqe->status. */ static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va, netdev_features_t dev_features) { … } #if IS_ENABLED(CONFIG_IPV6) #define MLX4_CQE_STATUS_IP_ANY … #else #define MLX4_CQE_STATUS_IP_ANY … #endif struct mlx4_en_xdp_buff { … }; int mlx4_en_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp) { … } int mlx4_en_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, enum xdp_rss_hash_type *rss_type) { … } int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) { … } void mlx4_en_rx_irq(struct mlx4_cq *mcq) { … } /* Rx CQ polling - called by NAPI */ int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget) { … } void mlx4_en_calc_rx_buf(struct net_device *dev) { … } /* RSS related functions */ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn, struct mlx4_en_rx_ring *ring, enum mlx4_qp_state *state, struct mlx4_qp *qp) { … } int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) { … } void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv) { … } /* Allocate rx qp's and configure them according to rss map */ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) { … } void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv) { … }
Codebase Browser
linux