linux/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

/*
 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/tcp.h>
#include <linux/bitmap.h>
#include <linux/filter.h>
#include <net/ip6_checksum.h>
#include <net/page_pool/helpers.h>
#include <net/inet_ecn.h>
#include <net/gro.h>
#include <net/udp.h>
#include <net/tcp.h>
#include <net/xdp_sock_drv.h>
#include "en.h"
#include "en/txrx.h"
#include "en_tc.h"
#include "eswitch.h"
#include "en_rep.h"
#include "en/rep/tc.h"
#include "ipoib/ipoib.h"
#include "en_accel/ipsec.h"
#include "en_accel/macsec.h"
#include "en_accel/ipsec_rxtx.h"
#include "en_accel/ktls_txrx.h"
#include "en/xdp.h"
#include "en/xsk/rx.h"
#include "en/health.h"
#include "en/params.h"
#include "devlink.h"
#include "en/devlink.h"

static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
				struct mlx5_cqe64 *cqe, u16 cqe_bcnt, u32 head_offset,
				u32 page_idx);
static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
				   struct mlx5_cqe64 *cqe, u16 cqe_bcnt, u32 head_offset,
				   u32 page_idx);
static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);

const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic = …;

static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq,
				       u32 cqcc, void *data)
{ … }

static void mlx5e_read_enhanced_title_slot(struct mlx5e_rq *rq,
					   struct mlx5_cqe64 *cqe)
{ … }

static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
					 struct mlx5_cqwq *wq,
					 u32 cqcc)
{ … }

static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq *wq,
					    struct mlx5e_cq_decomp *cqd,
					    u32 cqcc)
{ … }

static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq *wq, int n)
{ … }

static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
					struct mlx5_cqwq *wq,
					u32 cqcc)
{ … }

static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
						struct mlx5_cqwq *wq,
						u32 cqcc)
{ … }

static u32 mlx5e_decompress_enhanced_cqe(struct mlx5e_rq *rq,
					 struct mlx5_cqwq *wq,
					 struct mlx5_cqe64 *cqe,
					 int budget_rem)
{ … }

static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
					     struct mlx5_cqwq *wq,
					     int update_owner_only,
					     int budget_rem)
{ … }

static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
					      struct mlx5_cqwq *wq,
					      int budget_rem)
{ … }

#define MLX5E_PAGECNT_BIAS_MAX …

static int mlx5e_page_alloc_fragmented(struct mlx5e_rq *rq,
				       struct mlx5e_frag_page *frag_page)
{ … }

static void mlx5e_page_release_fragmented(struct mlx5e_rq *rq,
					  struct mlx5e_frag_page *frag_page)
{ … }

static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
				    struct mlx5e_wqe_frag_info *frag)
{ … }

static bool mlx5e_frag_can_release(struct mlx5e_wqe_frag_info *frag)
{ … }

static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
				     struct mlx5e_wqe_frag_info *frag)
{ … }

static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
{ … }

static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
			      u16 ix)
{ … }

static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
				     struct mlx5e_wqe_frag_info *wi)
{ … }

static void mlx5e_xsk_free_rx_wqe(struct mlx5e_wqe_frag_info *wi)
{ … }

static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
{ … }

static void mlx5e_xsk_free_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
{ … }

static void mlx5e_free_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
{ … }

static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
{ … }

static int mlx5e_refill_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
{ … }

static void
mlx5e_add_skb_shared_info_frag(struct mlx5e_rq *rq, struct skb_shared_info *sinfo,
			       struct xdp_buff *xdp, struct mlx5e_frag_page *frag_page,
			       u32 frag_offset, u32 len)
{ … }

static inline void
mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
		   struct mlx5e_frag_page *frag_page,
		   u32 frag_offset, u32 len,
		   unsigned int truesize)
{ … }

static inline void
mlx5e_copy_skb_header(struct mlx5e_rq *rq, struct sk_buff *skb,
		      struct page *page, dma_addr_t addr,
		      int offset_from, int dma_offset, u32 headlen)
{ … }

static void
mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
{ … }

static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
{ … }

/* This function returns the size of the continuous free space inside a bitmap
 * that starts from first and no longer than len including circular ones.
 */
static int bitmap_find_window(unsigned long *bitmap, int len,
			      int bitmap_size, int first)
{ … }

static void build_ksm_umr(struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *umr_wqe,
			  __be32 key, u16 offset, u16 ksm_len)
{ … }

static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
				     struct mlx5e_icosq *sq,
				     u16 ksm_entries, u16 index)
{ … }

static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq)
{ … }

static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
{ … }

static void
mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index)
{ … }

void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq)
{ … }

static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
{ … }

INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
{ … }

void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq)
{ … }

static void mlx5e_handle_shampo_hd_umr(struct mlx5e_shampo_umr umr,
				       struct mlx5e_icosq *sq)
{ … }

int mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
{ … }

INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
{ … }

static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp)
{ … }

static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
				 u32 cqe_bcnt)
{ … }

static void *mlx5e_shampo_get_packet_hd(struct mlx5e_rq *rq, u16 header_index)
{ … }

static void mlx5e_shampo_update_ipv4_udp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4)
{ … }

static void mlx5e_shampo_update_ipv6_udp_hdr(struct mlx5e_rq *rq, struct ipv6hdr *ipv6)
{ … }

static void mlx5e_shampo_update_fin_psh_flags(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
					      struct tcphdr *skb_tcp_hd)
{ … }

static void mlx5e_shampo_update_ipv4_tcp_hdr(struct mlx5e_rq *rq, struct iphdr *ipv4,
					     struct mlx5_cqe64 *cqe, bool match)
{ … }

static void mlx5e_shampo_update_ipv6_tcp_hdr(struct mlx5e_rq *rq, struct ipv6hdr *ipv6,
					     struct mlx5_cqe64 *cqe, bool match)
{ … }

static void mlx5e_shampo_update_hdr(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
{ … }

static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
				      struct sk_buff *skb)
{ … }

static inline bool is_last_ethertype_ip(struct sk_buff *skb, int *network_depth,
					__be16 *proto)
{ … }

static inline void mlx5e_enable_ecn(struct mlx5e_rq *rq, struct sk_buff *skb)
{ … }

static u8 get_ip_proto(struct sk_buff *skb, int network_depth, __be16 proto)
{ … }

#define short_frame(size) …

#define MAX_PADDING …

static void
tail_padding_csum_slow(struct sk_buff *skb, int offset, int len,
		       struct mlx5e_rq_stats *stats)
{ … }

static void
tail_padding_csum(struct sk_buff *skb, int offset,
		  struct mlx5e_rq_stats *stats)
{ … }

static void
mlx5e_skb_csum_fixup(struct sk_buff *skb, int network_depth, __be16 proto,
		     struct mlx5e_rq_stats *stats)
{ … }

static inline void mlx5e_handle_csum(struct net_device *netdev,
				     struct mlx5_cqe64 *cqe,
				     struct mlx5e_rq *rq,
				     struct sk_buff *skb,
				     bool   lro)
{ … }

#define MLX5E_CE_BIT_MASK …

static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
				      u32 cqe_bcnt,
				      struct mlx5e_rq *rq,
				      struct sk_buff *skb)
{ … }

static void mlx5e_shampo_complete_rx_cqe(struct mlx5e_rq *rq,
					 struct mlx5_cqe64 *cqe,
					 u32 cqe_bcnt,
					 struct sk_buff *skb)
{ … }

static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
					 struct mlx5_cqe64 *cqe,
					 u32 cqe_bcnt,
					 struct sk_buff *skb)
{ … }

static inline
struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
				       u32 frag_size, u16 headroom,
				       u32 cqe_bcnt, u32 metasize)
{ … }

static void mlx5e_fill_mxbuf(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
			     void *va, u16 headroom, u32 frame_sz, u32 len,
			     struct mlx5e_xdp_buff *mxbuf)
{ … }

static struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
			  struct mlx5_cqe64 *cqe, u32 cqe_bcnt)
{ … }

static struct sk_buff *
mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
			     struct mlx5_cqe64 *cqe, u32 cqe_bcnt)
{ … }

static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

static void mlx5e_handle_rx_err_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

#ifdef CONFIG_MLX5_ESWITCH
static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = …;
#endif

static void
mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
			   struct mlx5e_frag_page *frag_page,
			   u32 data_bcnt, u32 data_offset)
{ … }

static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
				   struct mlx5_cqe64 *cqe, u16 cqe_bcnt, u32 head_offset,
				   u32 page_idx)
{ … }

static struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
				struct mlx5_cqe64 *cqe, u16 cqe_bcnt, u32 head_offset,
				u32 page_idx)
{ … }

static struct sk_buff *
mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
			  struct mlx5_cqe64 *cqe, u16 header_index)
{ … }

static void
mlx5e_shampo_align_fragment(struct sk_buff *skb, u8 log_stride_sz)
{ … }

static void
mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
{ … }

static bool
mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt)
{ … }

static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

static int mlx5e_rx_cq_process_enhanced_cqe_comp(struct mlx5e_rq *rq,
						 struct mlx5_cqwq *cqwq,
						 int budget_rem)
{ … }

static int mlx5e_rx_cq_process_basic_cqe_comp(struct mlx5e_rq *rq,
					      struct mlx5_cqwq *cqwq,
					      int budget_rem)
{ … }

int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
{ … }

#ifdef CONFIG_MLX5_CORE_IPOIB

#define MLX5_IB_GRH_SGID_OFFSET …
#define MLX5_IB_GRH_DGID_OFFSET …
#define MLX5_GID_SIZE …

static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
					 struct mlx5_cqe64 *cqe,
					 u32 cqe_bcnt,
					 struct sk_buff *skb)
{ … }

static void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

const struct mlx5e_rx_handlers mlx5i_rx_handlers = …;
#endif /* CONFIG_MLX5_CORE_IPOIB */

int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk)
{ … }

static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{ … }

void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params)
{ … }