linux/drivers/net/ethernet/mellanox/mlx5/core/en_main.c

/*
 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/dim.h>
#include <net/tc_act/tc_gact.h>
#include <linux/mlx5/fs.h>
#include <net/vxlan.h>
#include <net/geneve.h>
#include <linux/bpf.h>
#include <linux/debugfs.h>
#include <linux/if_bridge.h>
#include <linux/filter.h>
#include <net/netdev_queues.h>
#include <net/page_pool/types.h>
#include <net/pkt_sched.h>
#include <net/xdp_sock_drv.h>
#include "eswitch.h"
#include "en.h"
#include "en/dim.h"
#include "en/txrx.h"
#include "en_tc.h"
#include "en_rep.h"
#include "en_accel/ipsec.h"
#include "en_accel/macsec.h"
#include "en_accel/en_accel.h"
#include "en_accel/ktls.h"
#include "lib/vxlan.h"
#include "lib/clock.h"
#include "en/port.h"
#include "en/xdp.h"
#include "lib/eq.h"
#include "en/monitor_stats.h"
#include "en/health.h"
#include "en/params.h"
#include "en/xsk/pool.h"
#include "en/xsk/setup.h"
#include "en/xsk/rx.h"
#include "en/xsk/tx.h"
#include "en/hv_vhca_stats.h"
#include "en/devlink.h"
#include "lib/mlx5.h"
#include "en/ptp.h"
#include "en/htb.h"
#include "qos.h"
#include "en/trap.h"
#include "lib/devcom.h"
#include "lib/sd.h"

static bool mlx5e_hw_gro_supported(struct mlx5_core_dev *mdev)
{ … }

bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
					    enum mlx5e_mpwrq_umr_mode umr_mode)
{ … }

void mlx5e_update_carrier(struct mlx5e_priv *priv)
{ … }

static void mlx5e_update_carrier_work(struct work_struct *work)
{ … }

static void mlx5e_update_stats_work(struct work_struct *work)
{ … }

void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
{ … }

static int async_event(struct notifier_block *nb, unsigned long event, void *data)
{ … }

static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
{ … }

static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
{ … }

static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data)
{ … }

static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data)
{ … }

static void mlx5e_devcom_cleanup_mpv(struct mlx5e_priv *priv)
{ … }

static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
{ … }

static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
{ … }

static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
{ … }

static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode)
{ … }

static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
				       struct mlx5e_icosq *sq,
				       struct mlx5e_umr_wqe *wqe)
{ … }

static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node)
{ … }

static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq)
{ … }

static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node)
{ … }

static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
{ … }

static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
{ … }


static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode)
{ … }

static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
				 u32 npages, u8 page_shift, u32 *umr_mkey,
				 dma_addr_t filler_addr,
				 enum mlx5e_mpwrq_umr_mode umr_mode,
				 u32 xsk_chunk_size)
{ … }

static int mlx5e_create_umr_ksm_mkey(struct mlx5_core_dev *mdev,
				     u64 nentries, u8 log_entry_size,
				     u32 *umr_mkey)
{ … }

static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
{ … }

static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
				       struct mlx5e_rq *rq)
{ … }

static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
{ … }

static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq)
{ … }

static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node)
{ … }

static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq)
{ … }

static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
{ … }

static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
{ … }

static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
{ … }

static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
			     u32 xdp_frag_size, struct mlx5e_rq *rq)
{ … }

static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
				struct mlx5e_params *params,
				struct mlx5e_rq_param *rqp,
				struct mlx5e_rq *rq,
				u32 *pool_size,
				int node)
{ … }

static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
{ … }

static int mlx5e_alloc_rq(struct mlx5e_params *params,
			  struct mlx5e_xsk_param *xsk,
			  struct mlx5e_rq_param *rqp,
			  int node, struct mlx5e_rq *rq)
{ … }

static void mlx5e_free_rq(struct mlx5e_rq *rq)
{ … }

int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter)
{ … }

static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
{ … }

static void mlx5e_flush_rq_cq(struct mlx5e_rq *rq)
{ … }

int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state)
{ … }

static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
{ … }

void mlx5e_destroy_rq(struct mlx5e_rq *rq)
{ … }

int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
{ … }

void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq)
{ … }

void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
{ … }

int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
		  struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
		  struct mlx5e_rq *rq)
{ … }

void mlx5e_activate_rq(struct mlx5e_rq *rq)
{ … }

void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
{ … }

void mlx5e_close_rq(struct mlx5e_rq *rq)
{ … }

u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev,
			   struct mlx5e_priv *priv,
			   const struct mlx5e_profile *profile,
			   u8 lag_port, u8 tc)
{ … }

static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
{ … }

static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
{ … }

static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
{ … }

static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
			     struct mlx5e_params *params,
			     struct xsk_buff_pool *xsk_pool,
			     struct mlx5e_sq_param *param,
			     struct mlx5e_xdpsq *sq,
			     bool is_redirect)
{ … }

static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
{ … }

static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
{ … }

static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
{ … }

static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
{ … }

static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work)
{ … }

static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
			     struct mlx5e_sq_param *param,
			     struct mlx5e_icosq *sq,
			     work_func_t recover_work_func)
{ … }

static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
{ … }

void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
{ … }

int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
{ … }

static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
			     int txq_ix,
			     struct mlx5e_params *params,
			     struct mlx5e_sq_param *param,
			     struct mlx5e_txqsq *sq,
			     int tc)
{ … }

void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
{ … }

static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
			   struct mlx5e_sq_param *param,
			   struct mlx5e_create_sq_param *csp,
			   u32 *sqn)
{ … }

int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
		    struct mlx5e_modify_sq_param *p)
{ … }

static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
{ … }

int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
			struct mlx5e_sq_param *param,
			struct mlx5e_create_sq_param *csp,
			u16 qos_queue_group_id,
			u32 *sqn)
{ … }

static int mlx5e_set_sq_maxrate(struct net_device *dev,
				struct mlx5e_txqsq *sq, u32 rate);

int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
		     struct mlx5e_params *params, struct mlx5e_sq_param *param,
		     struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
		     struct mlx5e_sq_stats *sq_stats)
{ … }

void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
{ … }

void mlx5e_tx_disable_queue(struct netdev_queue *txq)
{ … }

void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
{ … }

void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
{ … }

void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
{ … }

static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
{ … }

bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
			       bool dim_enabled)
{ … }

bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
					bool dim_enabled, bool keep_dim_state)
{ … }

static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
			    struct mlx5e_sq_param *param, struct mlx5e_icosq *sq,
			    work_func_t recover_work_func)
{ … }

void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
{ … }

void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
{ … }

static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
{ … }

int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
		     struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
		     struct mlx5e_xdpsq *sq, bool is_redirect)
{ … }

void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
{ … }

static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
				 struct net_device *netdev,
				 struct workqueue_struct *workqueue,
				 struct mlx5e_cq_param *param,
				 struct mlx5e_cq *cq)
{ … }

static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev,
			  struct mlx5e_cq_param *param,
			  struct mlx5e_create_cq_param *ccp,
			  struct mlx5e_cq *cq)
{ … }

static void mlx5e_free_cq(struct mlx5e_cq *cq)
{ … }

static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
{ … }

static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
{ … }

int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder,
		  struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
		  struct mlx5e_cq *cq)
{ … }

void mlx5e_close_cq(struct mlx5e_cq *cq)
{ … }

int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
				u8 cq_period_mode)
{ … }

int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
			       u16 cq_period, u16 cq_max_count, u8 cq_period_mode)
{ … }

static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
			     struct mlx5e_params *params,
			     struct mlx5e_create_cq_param *ccp,
			     struct mlx5e_channel_param *cparam)
{ … }

static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
{ … }

static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq)
{ … }

static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix,
					u32 *hw_id)
{ … }

static int mlx5e_open_sqs(struct mlx5e_channel *c,
			  struct mlx5e_params *params,
			  struct mlx5e_channel_param *cparam)
{ … }

static void mlx5e_close_sqs(struct mlx5e_channel *c)
{ … }

static int mlx5e_set_sq_maxrate(struct net_device *dev,
				struct mlx5e_txqsq *sq, u32 rate)
{ … }

static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
{ … }

static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
			     struct mlx5e_rq_param *rq_params)
{ … }

static int mlx5e_open_queues(struct mlx5e_channel *c,
			     struct mlx5e_params *params,
			     struct mlx5e_channel_param *cparam)
{ … }

static void mlx5e_close_queues(struct mlx5e_channel *c)
{ … }

static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
{ … }

static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
{ … }

void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
{ … }

void mlx5e_trigger_napi_sched(struct napi_struct *napi)
{ … }

static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
			      struct mlx5e_params *params,
			      struct mlx5e_channel_param *cparam,
			      struct xsk_buff_pool *xsk_pool,
			      struct mlx5e_channel **cp)
{ … }

static void mlx5e_activate_channel(struct mlx5e_channel *c)
{ … }

static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
{ … }

static void mlx5e_close_channel(struct mlx5e_channel *c)
{ … }

int mlx5e_open_channels(struct mlx5e_priv *priv,
			struct mlx5e_channels *chs)
{ … }

static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
{ … }

static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
{ … }

static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
{ … }

void mlx5e_close_channels(struct mlx5e_channels *chs)
{ … }

static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
{ … }

static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge);

static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
			 struct mlx5e_params *params, u16 mtu)
{ … }

static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
			    struct mlx5e_params *params, u16 *mtu)
{ … }

int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
{ … }

MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);

void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
{ … }

static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc,
				struct netdev_tc_txq *tc_to_txq)
{ … }

int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
{ … }

static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
					   struct mlx5e_params *params)
{ … }

static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv)
{ … }

MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_tc_and_tx_queues);

static int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
{ … }

MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);

static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
{ … }

void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
{ … }

static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv)
{ … }

void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
{ … }

static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
				    struct mlx5e_params *new_params,
				    mlx5e_fp_preactivate preactivate,
				    void *context)
{ … }

static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
				      struct mlx5e_channels *new_chs,
				      mlx5e_fp_preactivate preactivate,
				      void *context)
{ … }

int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
			     struct mlx5e_params *params,
			     mlx5e_fp_preactivate preactivate,
			     void *context, bool reset)
{ … }

int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
{ … }

void mlx5e_timestamp_init(struct mlx5e_priv *priv)
{ … }

static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
				     enum mlx5_port_status state)
{ … }

int mlx5e_open_locked(struct net_device *netdev)
{ … }

int mlx5e_open(struct net_device *netdev)
{ … }

int mlx5e_close_locked(struct net_device *netdev)
{ … }

int mlx5e_close(struct net_device *netdev)
{ … }

static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
{ … }

static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
			       struct mlx5e_rq *rq,
			       struct mlx5e_rq_param *param)
{ … }

static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
			       struct mlx5e_cq *cq,
			       struct mlx5e_cq_param *param)
{ … }

int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
		       struct mlx5e_rq *drop_rq)
{ … }

void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
{ … }

static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
{ … }

static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
{ … }

static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
						 int ntc, int nch)
{ … }

static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
					 struct tc_mqprio_qopt *qopt)
{ … }

static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc)
{ … }

static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params,
					  struct mlx5e_mqprio_rl *rl)
{ … }

static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params,
					    struct tc_mqprio_qopt_offload *mqprio,
					    struct mlx5e_mqprio_rl *rl)
{ … }

static void mlx5e_params_mqprio_reset(struct mlx5e_params *params)
{ … }

static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv,
				     struct tc_mqprio_qopt *mqprio)
{ … }

static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv,
					 struct tc_mqprio_qopt_offload *mqprio)
{ … }

static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[])
{ … }

static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev,
						      u8 num_tc, u64 max_rate[])
{ … }

static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv,
					 struct tc_mqprio_qopt_offload *mqprio)
{ … }

static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
				 struct tc_mqprio_qopt_offload *mqprio)
{ … }

static LIST_HEAD(mlx5e_block_cb_list);

static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
			  void *type_data)
{ … }

void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
{ … }

void
mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
{ … }

static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv)
{ … }

static void mlx5e_set_rx_mode(struct net_device *dev)
{ … }

static int mlx5e_set_mac(struct net_device *netdev, void *addr)
{ … }

#define MLX5E_SET_FEATURE(features, feature, enable) …

mlx5e_feature_handler;

static int set_feature_lro(struct net_device *netdev, bool enable)
{ … }

static int set_feature_hw_gro(struct net_device *netdev, bool enable)
{ … }

static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
{ … }

static int set_feature_hw_tc(struct net_device *netdev, bool enable)
{ … }

static int set_feature_rx_all(struct net_device *netdev, bool enable)
{ … }

static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
{ … }

bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
			       bool dim_enabled)
{ … }

bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
					bool dim_enabled, bool keep_dim_state)
{ … }

static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable)
{ … }

static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx)
{ … }

static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
{ … }

static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
{ … }

int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
{ … }

int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
{ … }

#ifdef CONFIG_MLX5_EN_ARFS
static int set_feature_arfs(struct net_device *netdev, bool enable)
{ … }
#endif

static int mlx5e_handle_feature(struct net_device *netdev,
				netdev_features_t *features,
				netdev_features_t feature,
				mlx5e_feature_handler feature_handler)
{ … }

void mlx5e_set_xdp_feature(struct net_device *netdev)
{ … }

int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
{ … }

static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev,
						       netdev_features_t features)
{ … }

static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
					    netdev_features_t features)
{ … }

static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
				   struct mlx5e_channels *chs,
				   struct mlx5e_params *new_params,
				   struct mlx5_core_dev *mdev)
{ … }

static bool mlx5e_params_validate_xdp(struct net_device *netdev,
				      struct mlx5_core_dev *mdev,
				      struct mlx5e_params *params)
{ … }

int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
		     mlx5e_fp_preactivate preactivate)
{ … }

static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
{ … }

int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx)
{ … }

static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter)
{ … }

static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx)
{ … }

int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
{ … }

int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
{ … }

static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
{ … }

#ifdef CONFIG_MLX5_ESWITCH
int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
{ … }

static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
			     __be16 vlan_proto)
{ … }

static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
{ … }

static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
{ … }

int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
		      int max_tx_rate)
{ … }

static int mlx5_vport_link2ifla(u8 esw_link)
{ … }

static int mlx5_ifla_link2vport(u8 ifla_link)
{ … }

static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
				   int link_state)
{ … }

int mlx5e_get_vf_config(struct net_device *dev,
			int vf, struct ifla_vf_info *ivi)
{ … }

int mlx5e_get_vf_stats(struct net_device *dev,
		       int vf, struct ifla_vf_stats *vf_stats)
{ … }

static bool
mlx5e_has_offload_stats(const struct net_device *dev, int attr_id)
{ … }

static int
mlx5e_get_offload_stats(int attr_id, const struct net_device *dev,
			void *sp)
{ … }
#endif

static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type)
{ … }

static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
							   struct sk_buff *skb)
{ … }

static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
						     struct sk_buff *skb,
						     netdev_features_t features)
{ … }

netdev_features_t mlx5e_features_check(struct sk_buff *skb,
				       struct net_device *netdev,
				       netdev_features_t features)
{ … }

static void mlx5e_tx_timeout_work(struct work_struct *work)
{ … }

static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
{ … }

static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev,
			     struct mlx5e_params *params)
{ … }

static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog)
{ … }

static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
{ … }

static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{ … }

#ifdef CONFIG_MLX5_ESWITCH
static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
				struct net_device *dev, u32 filter_mask,
				int nlflags)
{ … }

static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
				u16 flags, struct netlink_ext_ack *extack)
{ … }
#endif

const struct net_device_ops mlx5e_netdev_ops = …;

static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
{ … }

void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu)
{ … }

static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
{ … }

static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table,
				unsigned int entry, struct udp_tunnel_info *ti)
{ … }

static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table,
				  unsigned int entry, struct udp_tunnel_info *ti)
{ … }

void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv)
{ … }

static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev)
{ … }

static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i,
				     struct netdev_queue_stats_rx *stats)
{ … }

static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i,
				     struct netdev_queue_stats_tx *stats)
{ … }

static void mlx5e_get_base_stats(struct net_device *dev,
				 struct netdev_queue_stats_rx *rx,
				 struct netdev_queue_stats_tx *tx)
{ … }

static const struct netdev_stat_ops mlx5e_stat_ops = …;

static void mlx5e_build_nic_netdev(struct net_device *netdev)
{ … }

void mlx5e_create_q_counters(struct mlx5e_priv *priv)
{ … }

void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
{ … }

static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
			  struct net_device *netdev)
{ … }

static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
{ … }

static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
{ … }

static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
{ … }

static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv)
{ … }

static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
{ … }

static void mlx5e_nic_enable(struct mlx5e_priv *priv)
{ … }

static void mlx5e_nic_disable(struct mlx5e_priv *priv)
{ … }

static int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
{ … }

static const struct mlx5e_profile mlx5e_nic_profile = …;

static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev,
					  const struct mlx5e_profile *profile)
{ … }

static unsigned int
mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev,
		   const struct mlx5e_profile *profile)

{ … }

int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev)
{ … }

void mlx5e_set_rx_mode_work(struct work_struct *work)
{ … }

/* mlx5e generic netdev management API (move to en_common.c) */
int mlx5e_priv_init(struct mlx5e_priv *priv,
		    const struct mlx5e_profile *profile,
		    struct net_device *netdev,
		    struct mlx5_core_dev *mdev)
{ … }

void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
{ … }

static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev,
					   const struct mlx5e_profile *profile)
{ … }

static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev,
					   const struct mlx5e_profile *profile)
{ … }

struct net_device *
mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile)
{ … }

static void mlx5e_update_features(struct net_device *netdev)
{ … }

static void mlx5e_reset_channels(struct net_device *netdev)
{ … }

int mlx5e_attach_netdev(struct mlx5e_priv *priv)
{ … }

void mlx5e_detach_netdev(struct mlx5e_priv *priv)
{ … }

static int
mlx5e_netdev_init_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
			  const struct mlx5e_profile *new_profile, void *new_ppriv)
{ … }

static int
mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
			    const struct mlx5e_profile *new_profile, void *new_ppriv)
{ … }

int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
				const struct mlx5e_profile *new_profile, void *new_ppriv)
{ … }

void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv)
{ … }

void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
{ … }

static int _mlx5e_resume(struct auxiliary_device *adev)
{ … }

static int mlx5e_resume(struct auxiliary_device *adev)
{ … }

static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg)
{ … }

static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
{ … }

static int _mlx5e_probe(struct auxiliary_device *adev)
{ … }

static int mlx5e_probe(struct auxiliary_device *adev,
		       const struct auxiliary_device_id *id)
{ … }

static void _mlx5e_remove(struct auxiliary_device *adev)
{ … }

static void mlx5e_remove(struct auxiliary_device *adev)
{ … }

static const struct auxiliary_device_id mlx5e_id_table[] = …;

MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table);

static struct auxiliary_driver mlx5e_driver = …;

int mlx5e_init(void)
{ … }

void mlx5e_cleanup(void)
{ … }