// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2019 Mellanox Technologies. */ #include <net/netfilter/nf_conntrack.h> #include <net/netfilter/nf_conntrack_core.h> #include <net/netfilter/nf_conntrack_zones.h> #include <net/netfilter/nf_conntrack_labels.h> #include <net/netfilter/nf_conntrack_helper.h> #include <net/netfilter/nf_conntrack_acct.h> #include <uapi/linux/tc_act/tc_pedit.h> #include <net/tc_act/tc_ct.h> #include <net/flow_offload.h> #include <net/netfilter/nf_flow_table.h> #include <linux/workqueue.h> #include <linux/refcount.h> #include <linux/xarray.h> #include <linux/if_macvlan.h> #include <linux/debugfs.h> #include "lib/fs_chains.h" #include "en/tc_ct.h" #include "en/tc/ct_fs.h" #include "en/tc_priv.h" #include "en/mod_hdr.h" #include "en/mapping.h" #include "en/tc/post_act.h" #include "en.h" #include "en_tc.h" #include "en_rep.h" #include "fs_core.h" #define MLX5_CT_STATE_ESTABLISHED_BIT … #define MLX5_CT_STATE_TRK_BIT … #define MLX5_CT_STATE_NAT_BIT … #define MLX5_CT_STATE_REPLY_BIT … #define MLX5_CT_STATE_RELATED_BIT … #define MLX5_CT_STATE_INVALID_BIT … #define MLX5_CT_STATE_NEW_BIT … #define MLX5_CT_LABELS_BITS … #define MLX5_CT_LABELS_MASK … /* Statically allocate modify actions for * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10. * This will be increased dynamically if needed (for the ipv6 snat + dnat). */ #define MLX5_CT_MIN_MOD_ACTS … #define ct_dbg(fmt, args...) … struct mlx5_tc_ct_debugfs { … }; struct mlx5_tc_ct_priv { … }; struct mlx5_ct_zone_rule { … }; struct mlx5_tc_ct_pre { … }; struct mlx5_ct_ft { … }; struct mlx5_ct_tuple { … }; struct mlx5_ct_counter { … }; enum { … }; struct mlx5_ct_entry { … }; static void mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_flow_attr *attr, struct mlx5e_mod_hdr_handle *mh); static const struct rhashtable_params cts_ht_params = …; static const struct rhashtable_params zone_params = …; static const struct rhashtable_params tuples_ht_params = …; static const struct rhashtable_params tuples_nat_ht_params = …; static bool mlx5_tc_ct_entry_in_ct_table(struct mlx5_ct_entry *entry) { … } static bool mlx5_tc_ct_entry_in_ct_nat_table(struct mlx5_ct_entry *entry) { … } static int mlx5_get_label_mapping(struct mlx5_tc_ct_priv *ct_priv, u32 *labels, u32 *id) { … } static void mlx5_put_label_mapping(struct mlx5_tc_ct_priv *ct_priv, u32 id) { … } static int mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple *tuple, struct flow_rule *rule) { … } static int mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple *tuple, struct flow_rule *rule) { … } static int mlx5_tc_ct_get_flow_source_match(struct mlx5_tc_ct_priv *ct_priv, struct net_device *ndev) { … } static int mlx5_tc_ct_set_tuple_match(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_flow_spec *spec, struct flow_rule *rule) { … } static void mlx5_tc_ct_counter_put(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry) { … } static void mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry, bool nat) { … } static void mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry) { … } static struct flow_action_entry * mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule) { … } static int mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv, struct mlx5e_tc_mod_hdr_acts *mod_acts, u8 ct_state, u32 mark, u32 labels_id, u8 zone_restore_id) { … } static int mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act, char *modact) { … } static int mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, struct mlx5e_tc_mod_hdr_acts *mod_acts) { … } static int mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_flow_attr *attr, struct flow_rule *flow_rule, struct mlx5e_mod_hdr_handle **mh, u8 zone_restore_id, bool nat_table, bool has_nat) { … } static void mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_flow_attr *attr, struct mlx5e_mod_hdr_handle *mh) { … } static int mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, struct mlx5_ct_entry *entry, bool nat, u8 zone_restore_id) { … } static int mlx5_tc_ct_entry_replace_rule(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, struct mlx5_ct_entry *entry, bool nat, u8 zone_restore_id) { … } static bool mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry) { … } static struct mlx5_ct_entry * mlx5_tc_ct_entry_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_tuple *tuple) { … } static void mlx5_tc_ct_entry_remove_from_tuples(struct mlx5_ct_entry *entry) { … } static void mlx5_tc_ct_entry_del(struct mlx5_ct_entry *entry) { … } static void mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry) { … } static void mlx5_tc_ct_entry_del_work(struct work_struct *work) { … } static void __mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry) { … } static struct mlx5_ct_counter * mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv *ct_priv) { … } static struct mlx5_ct_counter * mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry) { … } static int mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, struct mlx5_ct_entry *entry, u8 zone_restore_id) { … } static int mlx5_tc_ct_entry_replace_rules(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, struct mlx5_ct_entry *entry, u8 zone_restore_id) { … } static int mlx5_tc_ct_block_flow_offload_replace(struct mlx5_ct_ft *ft, struct flow_rule *flow_rule, struct mlx5_ct_entry *entry, unsigned long cookie) { … } static int mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, struct flow_cls_offload *flow) { … } static int mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft, struct flow_cls_offload *flow) { … } static int mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft, struct flow_cls_offload *f) { … } static int mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data, void *cb_priv) { … } static bool mlx5_tc_ct_skb_to_tuple(struct sk_buff *skb, struct mlx5_ct_tuple *tuple, u16 zone) { … } int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec) { … } void mlx5_tc_ct_match_del(struct mlx5_tc_ct_priv *priv, struct mlx5_ct_attr *ct_attr) { … } int mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_spec *spec, struct flow_cls_offload *f, struct mlx5_ct_attr *ct_attr, struct netlink_ext_ack *extack) { … } int mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr, const struct flow_action_entry *act, struct netlink_ext_ack *extack) { … } static int tc_ct_pre_ct_add_rules(struct mlx5_ct_ft *ct_ft, struct mlx5_tc_ct_pre *pre_ct, bool nat) { … } static void tc_ct_pre_ct_del_rules(struct mlx5_ct_ft *ct_ft, struct mlx5_tc_ct_pre *pre_ct) { … } static int mlx5_tc_ct_alloc_pre_ct(struct mlx5_ct_ft *ct_ft, struct mlx5_tc_ct_pre *pre_ct, bool nat) { … } static void mlx5_tc_ct_free_pre_ct(struct mlx5_ct_ft *ct_ft, struct mlx5_tc_ct_pre *pre_ct) { … } static int mlx5_tc_ct_alloc_pre_ct_tables(struct mlx5_ct_ft *ft) { … } static void mlx5_tc_ct_free_pre_ct_tables(struct mlx5_ct_ft *ft) { … } /* To avoid false lock dependency warning set the ct_entries_ht lock * class different than the lock class of the ht being used when deleting * last flow from a group and then deleting a group, we get into del_sw_flow_group() * which call rhashtable_destroy on fg->ftes_hash which will take ht->mutex but * it's different than the ht->mutex here. */ static struct lock_class_key ct_entries_ht_lock_key; static struct mlx5_ct_ft * mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone, struct nf_flowtable *nf_ft) { … } static void mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg) { … } static void mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft) { … } /* We translate the tc filter with CT action to the following HW model: * * +-----------------------+ * + rule (either original + * + or post_act rule) + * +-----------------------+ * | set act_miss_cookie mapping * | set fte_id * | set tunnel_id * | rest of actions before the CT action (for this orig/post_act rule) * | * +-------------+ * | Chain 0 | * | optimization| * | v * | +---------------------+ * | + pre_ct/pre_ct_nat + if matches +----------------------+ * | + zone+nat match +---------------->+ post_act (see below) + * | +---------------------+ set zone +----------------------+ * | | * +-------------+ set zone * | * v * +--------------------+ * + CT (nat or no nat) + * + tuple + zone match + * +--------------------+ * | set mark * | set labels_id * | set established * | set zone_restore * | do nat (if needed) * v * +--------------+ * + post_act + rest of parsed filter's actions * + fte_id match +------------------------> * +--------------+ * */ static int __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_flow_attr *attr) { … } int mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr) { … } static void __mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_flow_attr *attr) { … } void mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr) { … } static int mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv) { … } static int mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch *esw, const char **err_msg) { … } static int mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv, enum mlx5_flow_namespace_type ns_type, struct mlx5e_post_act *post_act) { … } static void mlx5_ct_tc_create_dbgfs(struct mlx5_tc_ct_priv *ct_priv) { … } static void mlx5_ct_tc_remove_dbgfs(struct mlx5_tc_ct_priv *ct_priv) { … } static struct mlx5_flow_handle * tc_ct_add_miss_rule(struct mlx5_flow_table *ft, struct mlx5_flow_table *next_ft) { … } static int tc_ct_add_ct_table_miss_rule(struct mlx5_flow_table *from, struct mlx5_flow_table *to, struct mlx5_flow_group **miss_group, struct mlx5_flow_handle **miss_rule) { … } static void tc_ct_del_ct_table_miss_rule(struct mlx5_flow_group *miss_group, struct mlx5_flow_handle *miss_rule) { … } #define INIT_ERR_PREFIX … struct mlx5_tc_ct_priv * mlx5_tc_ct_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, struct mod_hdr_tbl *mod_hdr, enum mlx5_flow_namespace_type ns_type, struct mlx5e_post_act *post_act) { … } void mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv) { … } bool mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv *ct_priv, struct sk_buff *skb, u8 zone_restore_id) { … }
Codebase Browser
linux