linux/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Mellanox Technologies. */

#include <linux/types.h>
#include <linux/crc32.h>
#include "dr_ste.h"

struct dr_hw_ste_format { … };

static u32 dr_ste_crc32_calc(const void *input_data, size_t length)
{ … }

bool mlx5dr_ste_supp_ttl_cs_recalc(struct mlx5dr_cmd_caps *caps)
{ … }

u32 mlx5dr_ste_calc_hash_index(u8 *hw_ste_p, struct mlx5dr_ste_htbl *htbl)
{ … }

u16 mlx5dr_ste_conv_bit_to_byte_mask(u8 *bit_mask)
{ … }

static u8 *dr_ste_get_tag(u8 *hw_ste_p)
{ … }

void mlx5dr_ste_set_bit_mask(u8 *hw_ste_p, u8 *bit_mask)
{ … }

static void dr_ste_set_always_hit(struct dr_hw_ste_format *hw_ste)
{ … }

static void dr_ste_set_always_miss(struct dr_hw_ste_format *hw_ste)
{ … }

bool mlx5dr_ste_is_miss_addr_set(struct mlx5dr_ste_ctx *ste_ctx,
				 u8 *hw_ste_p)
{ … }

void mlx5dr_ste_set_miss_addr(struct mlx5dr_ste_ctx *ste_ctx,
			      u8 *hw_ste_p, u64 miss_addr)
{ … }

static void dr_ste_always_miss_addr(struct mlx5dr_ste_ctx *ste_ctx,
				    u8 *hw_ste, u64 miss_addr)
{ … }

void mlx5dr_ste_set_hit_addr(struct mlx5dr_ste_ctx *ste_ctx,
			     u8 *hw_ste, u64 icm_addr, u32 ht_size)
{ … }

u64 mlx5dr_ste_get_icm_addr(struct mlx5dr_ste *ste)
{ … }

u64 mlx5dr_ste_get_mr_addr(struct mlx5dr_ste *ste)
{ … }

u8 *mlx5dr_ste_get_hw_ste(struct mlx5dr_ste *ste)
{ … }

struct list_head *mlx5dr_ste_get_miss_list(struct mlx5dr_ste *ste)
{ … }

static void dr_ste_always_hit_htbl(struct mlx5dr_ste_ctx *ste_ctx,
				   u8 *hw_ste,
				   struct mlx5dr_ste_htbl *next_htbl)
{ … }

bool mlx5dr_ste_is_last_in_rule(struct mlx5dr_matcher_rx_tx *nic_matcher,
				u8 ste_location)
{ … }

/* Replace relevant fields, except of:
 * htbl - keep the origin htbl
 * miss_list + list - already took the src from the list.
 * icm_addr/mr_addr - depends on the hosting table.
 *
 * Before:
 * | a | -> | b | -> | c | ->
 *
 * After:
 * | a | -> | c | ->
 * While the data that was in b copied to a.
 */
static void dr_ste_replace(struct mlx5dr_ste *dst, struct mlx5dr_ste *src)
{ … }

/* Free ste which is the head and the only one in miss_list */
static void
dr_ste_remove_head_ste(struct mlx5dr_ste_ctx *ste_ctx,
		       struct mlx5dr_ste *ste,
		       struct mlx5dr_matcher_rx_tx *nic_matcher,
		       struct mlx5dr_ste_send_info *ste_info_head,
		       struct list_head *send_ste_list,
		       struct mlx5dr_ste_htbl *stats_tbl)
{ … }

/* Free ste which is the head but NOT the only one in miss_list:
 * |_ste_| --> |_next_ste_| -->|__| -->|__| -->/0
 */
static void
dr_ste_replace_head_ste(struct mlx5dr_matcher_rx_tx *nic_matcher,
			struct mlx5dr_ste *ste,
			struct mlx5dr_ste *next_ste,
			struct mlx5dr_ste_send_info *ste_info_head,
			struct list_head *send_ste_list,
			struct mlx5dr_ste_htbl *stats_tbl)

{ … }

/* Free ste that is located in the middle of the miss list:
 * |__| -->|_prev_ste_|->|_ste_|-->|_next_ste_|
 */
static void dr_ste_remove_middle_ste(struct mlx5dr_ste_ctx *ste_ctx,
				     struct mlx5dr_ste *ste,
				     struct mlx5dr_ste_send_info *ste_info,
				     struct list_head *send_ste_list,
				     struct mlx5dr_ste_htbl *stats_tbl)
{ … }

void mlx5dr_ste_free(struct mlx5dr_ste *ste,
		     struct mlx5dr_matcher *matcher,
		     struct mlx5dr_matcher_rx_tx *nic_matcher)
{ … }

bool mlx5dr_ste_equal_tag(void *src, void *dst)
{ … }

void mlx5dr_ste_set_hit_addr_by_next_htbl(struct mlx5dr_ste_ctx *ste_ctx,
					  u8 *hw_ste,
					  struct mlx5dr_ste_htbl *next_htbl)
{ … }

void mlx5dr_ste_prepare_for_postsend(struct mlx5dr_ste_ctx *ste_ctx,
				     u8 *hw_ste_p, u32 ste_size)
{ … }

/* Init one ste as a pattern for ste data array */
void mlx5dr_ste_set_formatted_ste(struct mlx5dr_ste_ctx *ste_ctx,
				  u16 gvmi,
				  enum mlx5dr_domain_nic_type nic_type,
				  struct mlx5dr_ste_htbl *htbl,
				  u8 *formatted_ste,
				  struct mlx5dr_htbl_connect_info *connect_info)
{ … }

int mlx5dr_ste_htbl_init_and_postsend(struct mlx5dr_domain *dmn,
				      struct mlx5dr_domain_rx_tx *nic_dmn,
				      struct mlx5dr_ste_htbl *htbl,
				      struct mlx5dr_htbl_connect_info *connect_info,
				      bool update_hw_ste)
{ … }

int mlx5dr_ste_create_next_htbl(struct mlx5dr_matcher *matcher,
				struct mlx5dr_matcher_rx_tx *nic_matcher,
				struct mlx5dr_ste *ste,
				u8 *cur_hw_ste,
				enum mlx5dr_icm_chunk_size log_table_size)
{ … }

struct mlx5dr_ste_htbl *mlx5dr_ste_htbl_alloc(struct mlx5dr_icm_pool *pool,
					      enum mlx5dr_icm_chunk_size chunk_size,
					      u16 lu_type, u16 byte_mask)
{ … }

int mlx5dr_ste_htbl_free(struct mlx5dr_ste_htbl *htbl)
{ … }

void mlx5dr_ste_set_actions_tx(struct mlx5dr_ste_ctx *ste_ctx,
			       struct mlx5dr_domain *dmn,
			       u8 *action_type_set,
			       u8 *hw_ste_arr,
			       struct mlx5dr_ste_actions_attr *attr,
			       u32 *added_stes)
{ … }

void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx,
			       struct mlx5dr_domain *dmn,
			       u8 *action_type_set,
			       u8 *hw_ste_arr,
			       struct mlx5dr_ste_actions_attr *attr,
			       u32 *added_stes)
{ … }

const struct mlx5dr_ste_action_modify_field *
mlx5dr_ste_conv_modify_hdr_sw_field(struct mlx5dr_ste_ctx *ste_ctx, u16 sw_field)
{ … }

void mlx5dr_ste_set_action_set(struct mlx5dr_ste_ctx *ste_ctx,
			       __be64 *hw_action,
			       u8 hw_field,
			       u8 shifter,
			       u8 length,
			       u32 data)
{ … }

void mlx5dr_ste_set_action_add(struct mlx5dr_ste_ctx *ste_ctx,
			       __be64 *hw_action,
			       u8 hw_field,
			       u8 shifter,
			       u8 length,
			       u32 data)
{ … }

void mlx5dr_ste_set_action_copy(struct mlx5dr_ste_ctx *ste_ctx,
				__be64 *hw_action,
				u8 dst_hw_field,
				u8 dst_shifter,
				u8 dst_len,
				u8 src_hw_field,
				u8 src_shifter)
{ … }

int mlx5dr_ste_set_action_decap_l3_list(struct mlx5dr_ste_ctx *ste_ctx,
					void *data, u32 data_sz,
					u8 *hw_action, u32 hw_action_sz,
					u16 *used_hw_action_num)
{ … }

static int
dr_ste_alloc_modify_hdr_chunk(struct mlx5dr_action *action)
{ … }

static void dr_ste_free_modify_hdr_chunk(struct mlx5dr_action *action)
{ … }

int mlx5dr_ste_alloc_modify_hdr(struct mlx5dr_action *action)
{ … }

void mlx5dr_ste_free_modify_hdr(struct mlx5dr_action *action)
{ … }

static int dr_ste_build_pre_check_spec(struct mlx5dr_domain *dmn,
				       struct mlx5dr_match_spec *spec)
{ … }

int mlx5dr_ste_build_pre_check(struct mlx5dr_domain *dmn,
			       u8 match_criteria,
			       struct mlx5dr_match_param *mask,
			       struct mlx5dr_match_param *value)
{ … }

int mlx5dr_ste_build_ste_arr(struct mlx5dr_matcher *matcher,
			     struct mlx5dr_matcher_rx_tx *nic_matcher,
			     struct mlx5dr_match_param *value,
			     u8 *ste_arr)
{ … }

#define IFC_GET_CLR(typ, p, fld, clear) …

#define memcpy_and_clear(to, from, len, clear) …

static void dr_ste_copy_mask_misc(char *mask, struct mlx5dr_match_misc *spec, bool clr)
{ … }

static void dr_ste_copy_mask_spec(char *mask, struct mlx5dr_match_spec *spec, bool clr)
{ … }

static void dr_ste_copy_mask_misc2(char *mask, struct mlx5dr_match_misc2 *spec, bool clr)
{ … }

static void dr_ste_copy_mask_misc3(char *mask, struct mlx5dr_match_misc3 *spec, bool clr)
{ … }

static void dr_ste_copy_mask_misc4(char *mask, struct mlx5dr_match_misc4 *spec, bool clr)
{ … }

static void dr_ste_copy_mask_misc5(char *mask, struct mlx5dr_match_misc5 *spec, bool clr)
{ … }

void mlx5dr_ste_copy_param(u8 match_criteria,
			   struct mlx5dr_match_param *set_param,
			   struct mlx5dr_match_parameters *mask,
			   bool clr)
{ … }

void mlx5dr_ste_build_eth_l2_src_dst(struct mlx5dr_ste_ctx *ste_ctx,
				     struct mlx5dr_ste_build *sb,
				     struct mlx5dr_match_param *mask,
				     bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l3_ipv6_dst(struct mlx5dr_ste_ctx *ste_ctx,
				      struct mlx5dr_ste_build *sb,
				      struct mlx5dr_match_param *mask,
				      bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l3_ipv6_src(struct mlx5dr_ste_ctx *ste_ctx,
				      struct mlx5dr_ste_build *sb,
				      struct mlx5dr_match_param *mask,
				      bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l3_ipv4_5_tuple(struct mlx5dr_ste_ctx *ste_ctx,
					  struct mlx5dr_ste_build *sb,
					  struct mlx5dr_match_param *mask,
					  bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l2_src(struct mlx5dr_ste_ctx *ste_ctx,
				 struct mlx5dr_ste_build *sb,
				 struct mlx5dr_match_param *mask,
				 bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l2_dst(struct mlx5dr_ste_ctx *ste_ctx,
				 struct mlx5dr_ste_build *sb,
				 struct mlx5dr_match_param *mask,
				 bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l2_tnl(struct mlx5dr_ste_ctx *ste_ctx,
				 struct mlx5dr_ste_build *sb,
				 struct mlx5dr_match_param *mask, bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l3_ipv4_misc(struct mlx5dr_ste_ctx *ste_ctx,
				       struct mlx5dr_ste_build *sb,
				       struct mlx5dr_match_param *mask,
				       bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_ipv6_l3_l4(struct mlx5dr_ste_ctx *ste_ctx,
				     struct mlx5dr_ste_build *sb,
				     struct mlx5dr_match_param *mask,
				     bool inner, bool rx)
{ … }

static int dr_ste_build_empty_always_hit_tag(struct mlx5dr_match_param *value,
					     struct mlx5dr_ste_build *sb,
					     u8 *tag)
{ … }

void mlx5dr_ste_build_empty_always_hit(struct mlx5dr_ste_build *sb, bool rx)
{ … }

void mlx5dr_ste_build_mpls(struct mlx5dr_ste_ctx *ste_ctx,
			   struct mlx5dr_ste_build *sb,
			   struct mlx5dr_match_param *mask,
			   bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_gre(struct mlx5dr_ste_ctx *ste_ctx,
			      struct mlx5dr_ste_build *sb,
			      struct mlx5dr_match_param *mask,
			      bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_mpls_over_gre(struct mlx5dr_ste_ctx *ste_ctx,
					struct mlx5dr_ste_build *sb,
					struct mlx5dr_match_param *mask,
					struct mlx5dr_cmd_caps *caps,
					bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_mpls_over_udp(struct mlx5dr_ste_ctx *ste_ctx,
					struct mlx5dr_ste_build *sb,
					struct mlx5dr_match_param *mask,
					struct mlx5dr_cmd_caps *caps,
					bool inner, bool rx)
{ … }

void mlx5dr_ste_build_icmp(struct mlx5dr_ste_ctx *ste_ctx,
			   struct mlx5dr_ste_build *sb,
			   struct mlx5dr_match_param *mask,
			   struct mlx5dr_cmd_caps *caps,
			   bool inner, bool rx)
{ … }

void mlx5dr_ste_build_general_purpose(struct mlx5dr_ste_ctx *ste_ctx,
				      struct mlx5dr_ste_build *sb,
				      struct mlx5dr_match_param *mask,
				      bool inner, bool rx)
{ … }

void mlx5dr_ste_build_eth_l4_misc(struct mlx5dr_ste_ctx *ste_ctx,
				  struct mlx5dr_ste_build *sb,
				  struct mlx5dr_match_param *mask,
				  bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_vxlan_gpe(struct mlx5dr_ste_ctx *ste_ctx,
				    struct mlx5dr_ste_build *sb,
				    struct mlx5dr_match_param *mask,
				    bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_geneve(struct mlx5dr_ste_ctx *ste_ctx,
				 struct mlx5dr_ste_build *sb,
				 struct mlx5dr_match_param *mask,
				 bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_geneve_tlv_opt(struct mlx5dr_ste_ctx *ste_ctx,
					 struct mlx5dr_ste_build *sb,
					 struct mlx5dr_match_param *mask,
					 struct mlx5dr_cmd_caps *caps,
					 bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_geneve_tlv_opt_exist(struct mlx5dr_ste_ctx *ste_ctx,
					       struct mlx5dr_ste_build *sb,
					       struct mlx5dr_match_param *mask,
					       struct mlx5dr_cmd_caps *caps,
					       bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_gtpu(struct mlx5dr_ste_ctx *ste_ctx,
			       struct mlx5dr_ste_build *sb,
			       struct mlx5dr_match_param *mask,
			       bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_gtpu_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
					     struct mlx5dr_ste_build *sb,
					     struct mlx5dr_match_param *mask,
					     struct mlx5dr_cmd_caps *caps,
					     bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_gtpu_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
					     struct mlx5dr_ste_build *sb,
					     struct mlx5dr_match_param *mask,
					     struct mlx5dr_cmd_caps *caps,
					     bool inner, bool rx)
{ … }

void mlx5dr_ste_build_register_0(struct mlx5dr_ste_ctx *ste_ctx,
				 struct mlx5dr_ste_build *sb,
				 struct mlx5dr_match_param *mask,
				 bool inner, bool rx)
{ … }

void mlx5dr_ste_build_register_1(struct mlx5dr_ste_ctx *ste_ctx,
				 struct mlx5dr_ste_build *sb,
				 struct mlx5dr_match_param *mask,
				 bool inner, bool rx)
{ … }

void mlx5dr_ste_build_src_gvmi_qpn(struct mlx5dr_ste_ctx *ste_ctx,
				   struct mlx5dr_ste_build *sb,
				   struct mlx5dr_match_param *mask,
				   struct mlx5dr_domain *dmn,
				   bool inner, bool rx)
{ … }

void mlx5dr_ste_build_flex_parser_0(struct mlx5dr_ste_ctx *ste_ctx,
				    struct mlx5dr_ste_build *sb,
				    struct mlx5dr_match_param *mask,
				    bool inner, bool rx)
{ … }

void mlx5dr_ste_build_flex_parser_1(struct mlx5dr_ste_ctx *ste_ctx,
				    struct mlx5dr_ste_build *sb,
				    struct mlx5dr_match_param *mask,
				    bool inner, bool rx)
{ … }

void mlx5dr_ste_build_tnl_header_0_1(struct mlx5dr_ste_ctx *ste_ctx,
				     struct mlx5dr_ste_build *sb,
				     struct mlx5dr_match_param *mask,
				     bool inner, bool rx)
{ … }

struct mlx5dr_ste_ctx *mlx5dr_ste_get_ctx(u8 version)
{ … }