#include <linux/mutex.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/vport.h>
#include <linux/mlx5/eswitch.h>
#include <net/devlink.h>
#include "mlx5_core.h"
#include "fs_core.h"
#include "fs_cmd.h"
#include "fs_ft_pool.h"
#include "diag/fs_tracepoint.h"
#include "devlink.h"
#define INIT_TREE_NODE_ARRAY_SIZE(...) …
#define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
...) …
#define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...) …\
#define ADD_NS(def_miss_act, ...) …
#define INIT_CAPS_ARRAY_SIZE(...) …
#define FS_CAP(cap) …
#define FS_REQUIRED_CAPS(...) …
#define FS_CHAINING_CAPS …
#define FS_CHAINING_CAPS_EGRESS …
#define FS_CHAINING_CAPS_RDMA_TX …
#define LEFTOVERS_NUM_LEVELS …
#define LEFTOVERS_NUM_PRIOS …
#define RDMA_RX_COUNTERS_PRIO_NUM_LEVELS …
#define RDMA_TX_COUNTERS_PRIO_NUM_LEVELS …
#define BY_PASS_PRIO_NUM_LEVELS …
#define BY_PASS_MIN_LEVEL …
#define KERNEL_RX_MACSEC_NUM_PRIOS …
#define KERNEL_RX_MACSEC_NUM_LEVELS …
#define KERNEL_RX_MACSEC_MIN_LEVEL …
#define ETHTOOL_PRIO_NUM_LEVELS …
#define ETHTOOL_NUM_PRIOS …
#define ETHTOOL_MIN_LEVEL …
#define KERNEL_NIC_PRIO_NUM_LEVELS …
#define KERNEL_NIC_NUM_PRIOS …
#define KERNEL_MIN_LEVEL …
#define KERNEL_NIC_TC_NUM_PRIOS …
#define KERNEL_NIC_TC_NUM_LEVELS …
#define ANCHOR_NUM_LEVELS …
#define ANCHOR_NUM_PRIOS …
#define ANCHOR_MIN_LEVEL …
#define OFFLOADS_MAX_FT …
#define OFFLOADS_NUM_PRIOS …
#define OFFLOADS_MIN_LEVEL …
#define LAG_PRIO_NUM_LEVELS …
#define LAG_NUM_PRIOS …
#define LAG_MIN_LEVEL …
#define KERNEL_TX_IPSEC_NUM_PRIOS …
#define KERNEL_TX_IPSEC_NUM_LEVELS …
#define KERNEL_TX_IPSEC_MIN_LEVEL …
#define KERNEL_TX_MACSEC_NUM_PRIOS …
#define KERNEL_TX_MACSEC_NUM_LEVELS …
#define KERNEL_TX_MACSEC_MIN_LEVEL …
struct node_caps { … };
static struct init_tree_node { … } root_fs = …;
static struct init_tree_node egress_root_fs = …;
enum { … };
#define RDMA_RX_IPSEC_NUM_PRIOS …
#define RDMA_RX_IPSEC_NUM_LEVELS …
#define RDMA_RX_IPSEC_MIN_LEVEL …
#define RDMA_RX_BYPASS_MIN_LEVEL …
#define RDMA_RX_KERNEL_MIN_LEVEL …
#define RDMA_RX_COUNTERS_MIN_LEVEL …
#define RDMA_RX_MACSEC_NUM_PRIOS …
#define RDMA_RX_MACSEC_PRIO_NUM_LEVELS …
#define RDMA_RX_MACSEC_MIN_LEVEL …
static struct init_tree_node rdma_rx_root_fs = …;
enum { … };
#define RDMA_TX_BYPASS_MIN_LEVEL …
#define RDMA_TX_COUNTERS_MIN_LEVEL …
#define RDMA_TX_IPSEC_NUM_PRIOS …
#define RDMA_TX_IPSEC_PRIO_NUM_LEVELS …
#define RDMA_TX_IPSEC_MIN_LEVEL …
#define RDMA_TX_MACSEC_NUM_PRIOS …
#define RDMA_TX_MACESC_PRIO_NUM_LEVELS …
#define RDMA_TX_MACSEC_MIN_LEVEL …
static struct init_tree_node rdma_tx_root_fs = …;
enum fs_i_lock_class { … };
static const struct rhashtable_params rhash_fte = …;
static const struct rhashtable_params rhash_fg = …;
static void del_hw_flow_table(struct fs_node *node);
static void del_hw_flow_group(struct fs_node *node);
static void del_hw_fte(struct fs_node *node);
static void del_sw_flow_table(struct fs_node *node);
static void del_sw_flow_group(struct fs_node *node);
static void del_sw_fte(struct fs_node *node);
static void del_sw_prio(struct fs_node *node);
static void del_sw_ns(struct fs_node *node);
static void del_sw_hw_rule(struct fs_node *node);
static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
struct mlx5_flow_destination *d2);
static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns);
static struct mlx5_flow_rule *
find_flow_rule(struct fs_fte *fte,
struct mlx5_flow_destination *dest);
static void tree_init_node(struct fs_node *node,
void (*del_hw_func)(struct fs_node *),
void (*del_sw_func)(struct fs_node *))
{ … }
static void tree_add_node(struct fs_node *node, struct fs_node *parent)
{ … }
static int tree_get_node(struct fs_node *node)
{ … }
static void nested_down_read_ref_node(struct fs_node *node,
enum fs_i_lock_class class)
{ … }
static void nested_down_write_ref_node(struct fs_node *node,
enum fs_i_lock_class class)
{ … }
static void down_write_ref_node(struct fs_node *node, bool locked)
{ … }
static void up_read_ref_node(struct fs_node *node)
{ … }
static void up_write_ref_node(struct fs_node *node, bool locked)
{ … }
static void tree_put_node(struct fs_node *node, bool locked)
{ … }
static int tree_remove_node(struct fs_node *node, bool locked)
{ … }
static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
unsigned int prio)
{ … }
static bool is_fwd_next_action(u32 action)
{ … }
static bool is_fwd_dest_type(enum mlx5_flow_destination_type type)
{ … }
static bool check_valid_spec(const struct mlx5_flow_spec *spec)
{ … }
struct mlx5_flow_root_namespace *find_root(struct fs_node *node)
{ … }
static inline struct mlx5_flow_steering *get_steering(struct fs_node *node)
{ … }
static inline struct mlx5_core_dev *get_dev(struct fs_node *node)
{ … }
static void del_sw_ns(struct fs_node *node)
{ … }
static void del_sw_prio(struct fs_node *node)
{ … }
static void del_hw_flow_table(struct fs_node *node)
{ … }
static void del_sw_flow_table(struct fs_node *node)
{ … }
static void modify_fte(struct fs_fte *fte)
{ … }
static void del_sw_hw_rule(struct fs_node *node)
{ … }
static void del_hw_fte(struct fs_node *node)
{ … }
static void del_sw_fte(struct fs_node *node)
{ … }
static void del_hw_flow_group(struct fs_node *node)
{ … }
static void del_sw_flow_group(struct fs_node *node)
{ … }
static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte)
{ … }
static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft,
const struct mlx5_flow_spec *spec,
struct mlx5_flow_act *flow_act)
{ … }
static void dealloc_flow_group(struct mlx5_flow_steering *steering,
struct mlx5_flow_group *fg)
{ … }
static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering,
u8 match_criteria_enable,
const void *match_criteria,
int start_index,
int end_index)
{ … }
static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *ft,
u8 match_criteria_enable,
const void *match_criteria,
int start_index,
int end_index,
struct list_head *prev)
{ … }
static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport,
enum fs_flow_table_type table_type,
enum fs_flow_table_op_mod op_mod,
u32 flags)
{ … }
static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node *root,
struct list_head *start,
bool reverse)
{ … }
static struct fs_node *find_prio_chains_parent(struct fs_node *parent,
struct fs_node **child)
{ … }
static struct mlx5_flow_table *find_closest_ft(struct fs_node *node, bool reverse,
bool skip)
{ … }
static struct mlx5_flow_table *find_next_chained_ft(struct fs_node *node)
{ … }
static struct mlx5_flow_table *find_prev_chained_ft(struct fs_node *node)
{ … }
static struct mlx5_flow_table *find_next_fwd_ft(struct mlx5_flow_table *ft,
struct mlx5_flow_act *flow_act)
{ … }
static int connect_fts_in_prio(struct mlx5_core_dev *dev,
struct fs_prio *prio,
struct mlx5_flow_table *ft)
{ … }
static struct mlx5_flow_table *find_closet_ft_prio_chains(struct fs_node *node,
struct fs_node *parent,
struct fs_node **child,
bool reverse)
{ … }
static int connect_prev_fts(struct mlx5_core_dev *dev,
struct mlx5_flow_table *ft,
struct fs_prio *prio)
{ … }
static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio
*prio)
{ … }
static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule,
struct mlx5_flow_destination *dest)
{ … }
int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle,
struct mlx5_flow_destination *new_dest,
struct mlx5_flow_destination *old_dest)
{ … }
static int connect_fwd_rules(struct mlx5_core_dev *dev,
struct mlx5_flow_table *new_next_ft,
struct mlx5_flow_table *old_next_ft)
{ … }
static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
struct fs_prio *prio)
{ … }
static void list_add_flow_table(struct mlx5_flow_table *ft,
struct fs_prio *prio)
{ … }
static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
struct mlx5_flow_table_attr *ft_attr,
enum fs_flow_table_op_mod op_mod,
u16 vport)
{ … }
struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
struct mlx5_flow_table_attr *ft_attr)
{ … }
EXPORT_SYMBOL(…);
u32 mlx5_flow_table_id(struct mlx5_flow_table *ft)
{ … }
EXPORT_SYMBOL(…);
struct mlx5_flow_table *
mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
struct mlx5_flow_table_attr *ft_attr, u16 vport)
{ … }
struct mlx5_flow_table*
mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
int prio, u32 level)
{ … }
EXPORT_SYMBOL(…);
#define MAX_FLOW_GROUP_SIZE …
struct mlx5_flow_table*
mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
struct mlx5_flow_table_attr *ft_attr)
{ … }
EXPORT_SYMBOL(…);
struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
u32 *fg_in)
{ … }
EXPORT_SYMBOL(…);
static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest)
{ … }
static struct mlx5_flow_handle *alloc_handle(int num_rules)
{ … }
static void destroy_flow_handle(struct fs_fte *fte,
struct mlx5_flow_handle *handle,
struct mlx5_flow_destination *dest,
int i)
{ … }
static struct mlx5_flow_handle *
create_flow_handle(struct fs_fte *fte,
struct mlx5_flow_destination *dest,
int dest_num,
int *modify_mask,
bool *new_rule)
{ … }
static struct mlx5_flow_handle *
add_rule_fte(struct fs_fte *fte,
struct mlx5_flow_group *fg,
struct mlx5_flow_destination *dest,
int dest_num,
bool update_action)
{ … }
static struct mlx5_flow_group *alloc_auto_flow_group(struct mlx5_flow_table *ft,
const struct mlx5_flow_spec *spec)
{ … }
static int create_auto_flow_group(struct mlx5_flow_table *ft,
struct mlx5_flow_group *fg)
{ … }
static bool mlx5_pkt_reformat_cmp(struct mlx5_pkt_reformat *p1,
struct mlx5_pkt_reformat *p2)
{ … }
static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
struct mlx5_flow_destination *d2)
{ … }
static struct mlx5_flow_rule *find_flow_rule(struct fs_fte *fte,
struct mlx5_flow_destination *dest)
{ … }
static bool check_conflicting_actions_vlan(const struct mlx5_fs_vlan *vlan0,
const struct mlx5_fs_vlan *vlan1)
{ … }
static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
const struct mlx5_flow_act *act2)
{ … }
static int check_conflicting_ftes(struct fs_fte *fte,
const struct mlx5_flow_context *flow_context,
const struct mlx5_flow_act *flow_act)
{ … }
static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg,
const struct mlx5_flow_spec *spec,
struct mlx5_flow_act *flow_act,
struct mlx5_flow_destination *dest,
int dest_num,
struct fs_fte *fte)
{ … }
static bool counter_is_valid(u32 action)
{ … }
static bool dest_is_valid(struct mlx5_flow_destination *dest,
struct mlx5_flow_act *flow_act,
struct mlx5_flow_table *ft)
{ … }
struct match_list { … };
static void free_match_list(struct match_list *head, bool ft_locked)
{ … }
static int build_match_list(struct match_list *match_head,
struct mlx5_flow_table *ft,
const struct mlx5_flow_spec *spec,
struct mlx5_flow_group *fg,
bool ft_locked)
{ … }
static u64 matched_fgs_get_version(struct list_head *match_head)
{ … }
static struct fs_fte *
lookup_fte_locked(struct mlx5_flow_group *g,
const u32 *match_value,
bool take_write)
{ … }
static struct mlx5_flow_handle *
try_add_to_existing_fg(struct mlx5_flow_table *ft,
struct list_head *match_head,
const struct mlx5_flow_spec *spec,
struct mlx5_flow_act *flow_act,
struct mlx5_flow_destination *dest,
int dest_num,
int ft_version)
{ … }
static struct mlx5_flow_handle *
_mlx5_add_flow_rules(struct mlx5_flow_table *ft,
const struct mlx5_flow_spec *spec,
struct mlx5_flow_act *flow_act,
struct mlx5_flow_destination *dest,
int dest_num)
{ … }
static bool fwd_next_prio_supported(struct mlx5_flow_table *ft)
{ … }
struct mlx5_flow_handle *
mlx5_add_flow_rules(struct mlx5_flow_table *ft,
const struct mlx5_flow_spec *spec,
struct mlx5_flow_act *flow_act,
struct mlx5_flow_destination *dest,
int num_dest)
{ … }
EXPORT_SYMBOL(…);
void mlx5_del_flow_rules(struct mlx5_flow_handle *handle)
{ … }
EXPORT_SYMBOL(…);
static struct mlx5_flow_table *find_next_ft(struct mlx5_flow_table *ft)
{ … }
static int update_root_ft_destroy(struct mlx5_flow_table *ft)
{ … }
static int disconnect_flow_table(struct mlx5_flow_table *ft)
{ … }
int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
{ … }
EXPORT_SYMBOL(…);
void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
{ … }
EXPORT_SYMBOL(…);
struct mlx5_flow_namespace *mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev,
int n)
{ … }
EXPORT_SYMBOL(…);
static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type)
{ … }
struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
enum mlx5_flow_namespace_type type)
{ … }
EXPORT_SYMBOL(…);
struct mlx5_flow_namespace *mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
enum mlx5_flow_namespace_type type,
int vport)
{ … }
static struct fs_prio *_fs_create_prio(struct mlx5_flow_namespace *ns,
unsigned int prio,
int num_levels,
enum fs_node_type type)
{ … }
static struct fs_prio *fs_create_prio_chained(struct mlx5_flow_namespace *ns,
unsigned int prio,
int num_levels)
{ … }
static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
unsigned int prio, int num_levels)
{ … }
static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
*ns)
{ … }
static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio,
int def_miss_act)
{ … }
static int create_leaf_prios(struct mlx5_flow_namespace *ns, int prio,
struct init_tree_node *prio_metadata)
{ … }
#define FLOW_TABLE_BIT_SZ …
#define GET_FLOW_TABLE_CAP(dev, offset) …
static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
{ … }
static int init_root_tree_recursive(struct mlx5_flow_steering *steering,
struct init_tree_node *init_node,
struct fs_node *fs_parent_node,
struct init_tree_node *init_parent_node,
int prio)
{ … }
static int init_root_tree(struct mlx5_flow_steering *steering,
struct init_tree_node *init_node,
struct fs_node *fs_parent_node)
{ … }
static void del_sw_root_ns(struct fs_node *node)
{ … }
static struct mlx5_flow_root_namespace
*create_root_ns(struct mlx5_flow_steering *steering,
enum fs_flow_table_type table_type)
{ … }
static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level);
static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level)
{ … }
static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level)
{ … }
static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns)
{ … }
#define ANCHOR_PRIO …
#define ANCHOR_SIZE …
#define ANCHOR_LEVEL …
static int create_anchor_flow_table(struct mlx5_flow_steering *steering)
{ … }
static int init_root_ns(struct mlx5_flow_steering *steering)
{ … }
static void clean_tree(struct fs_node *node)
{ … }
static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns)
{ … }
static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)
{ … }
static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)
{ … }
#define PORT_SEL_NUM_LEVELS …
static int init_port_sel_root_ns(struct mlx5_flow_steering *steering)
{ … }
static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering)
{ … }
static int init_rdma_tx_root_ns(struct mlx5_flow_steering *steering)
{ … }
static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
struct mlx5_flow_namespace *ns)
{ … }
static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
struct fs_prio *maj_prio)
{ … }
static int create_fdb_chains(struct mlx5_flow_steering *steering,
int fs_prio,
int chains)
{ … }
static int create_fdb_fast_path(struct mlx5_flow_steering *steering)
{ … }
static int create_fdb_bypass(struct mlx5_flow_steering *steering)
{ … }
static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering)
{ … }
static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
{ … }
static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
{ … }
static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
{ … }
int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports)
{ … }
void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev)
{ … }
int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports)
{ … }
void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev)
{ … }
u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type)
{ … }
static int init_egress_root_ns(struct mlx5_flow_steering *steering)
{ … }
static int mlx5_fs_mode_validate(struct devlink *devlink, u32 id,
union devlink_param_value val,
struct netlink_ext_ack *extack)
{ … }
static int mlx5_fs_mode_set(struct devlink *devlink, u32 id,
struct devlink_param_gset_ctx *ctx,
struct netlink_ext_ack *extack)
{ … }
static int mlx5_fs_mode_get(struct devlink *devlink, u32 id,
struct devlink_param_gset_ctx *ctx)
{ … }
static const struct devlink_param mlx5_fs_params[] = …;
void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev)
{ … }
int mlx5_fs_core_init(struct mlx5_core_dev *dev)
{ … }
void mlx5_fs_core_free(struct mlx5_core_dev *dev)
{ … }
int mlx5_fs_core_alloc(struct mlx5_core_dev *dev)
{ … }
int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
{ … }
EXPORT_SYMBOL(…);
int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
{ … }
EXPORT_SYMBOL(…);
static struct mlx5_flow_root_namespace
*get_root_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type ns_type)
{ … }
struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev,
u8 ns_type, u8 num_actions,
void *modify_actions)
{ … }
EXPORT_SYMBOL(…);
void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
struct mlx5_modify_hdr *modify_hdr)
{ … }
EXPORT_SYMBOL(…);
struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev,
struct mlx5_pkt_reformat_params *params,
enum mlx5_flow_namespace_type ns_type)
{ … }
EXPORT_SYMBOL(…);
void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
struct mlx5_pkt_reformat *pkt_reformat)
{ … }
EXPORT_SYMBOL(…);
int mlx5_get_match_definer_id(struct mlx5_flow_definer *definer)
{ … }
struct mlx5_flow_definer *
mlx5_create_match_definer(struct mlx5_core_dev *dev,
enum mlx5_flow_namespace_type ns_type, u16 format_id,
u32 *match_mask)
{ … }
void mlx5_destroy_match_definer(struct mlx5_core_dev *dev,
struct mlx5_flow_definer *definer)
{ … }
int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_root_namespace *peer_ns,
u16 peer_vhca_id)
{ … }
int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
enum mlx5_flow_steering_mode mode)
{ … }