linux/net/sched/cls_api.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/cls_api.c	Packet classifier API.
 *
 * Authors:	Alexey Kuznetsov, <[email protected]>
 *
 * Changes:
 *
 * Eduardo J. Blanco <[email protected]> :990222: kmod support
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/jhash.h>
#include <linux/rculist.h>
#include <linux/rhashtable.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_pedit.h>
#include <net/tc_act/tc_mirred.h>
#include <net/tc_act/tc_vlan.h>
#include <net/tc_act/tc_tunnel_key.h>
#include <net/tc_act/tc_csum.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_act/tc_police.h>
#include <net/tc_act/tc_sample.h>
#include <net/tc_act/tc_skbedit.h>
#include <net/tc_act/tc_ct.h>
#include <net/tc_act/tc_mpls.h>
#include <net/tc_act/tc_gate.h>
#include <net/flow_offload.h>
#include <net/tc_wrapper.h>

/* The list of all installed classifier types */
static LIST_HEAD(tcf_proto_base);

/* Protects list of registered TC modules. It is pure SMP lock. */
static DEFINE_RWLOCK(cls_mod_lock);

static struct xarray tcf_exts_miss_cookies_xa;
struct tcf_exts_miss_cookie_node { … };

/* Each tc action entry cookie will be comprised of 32bit miss_cookie_base +
 * action index in the exts tc actions array.
 */
tcf_exts_miss_cookie;

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
static int
tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
				u32 handle)
{ … }

static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
{ … }

static struct tcf_exts_miss_cookie_node *
tcf_exts_miss_cookie_lookup(u64 miss_cookie, int *act_index)
{ … }
#else /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
static int
tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
				u32 handle)
{
	return 0;
}

static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
{
}
#endif /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */

static u64 tcf_exts_miss_cookie_get(u32 miss_cookie_base, int act_index)
{ … }

#ifdef CONFIG_NET_CLS_ACT
DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc);
EXPORT_SYMBOL(…);

void tc_skb_ext_tc_enable(void)
{ … }
EXPORT_SYMBOL(…);

void tc_skb_ext_tc_disable(void)
{ … }
EXPORT_SYMBOL(…);
#endif

static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
{ … }

static void tcf_proto_signal_destroying(struct tcf_chain *chain,
					struct tcf_proto *tp)
{ … }

static bool tcf_proto_cmp(const struct tcf_proto *tp1,
			  const struct tcf_proto *tp2)
{ … }

static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
					struct tcf_proto *tp)
{ … }

static void
tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
{ … }

/* Find classifier type by string name */

static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
{ … }

static const struct tcf_proto_ops *
tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
		     struct netlink_ext_ack *extack)
{ … }

/* Register(unregister) new classifier type */

int register_tcf_proto_ops(struct tcf_proto_ops *ops)
{ … }
EXPORT_SYMBOL(…);

static struct workqueue_struct *tc_filter_wq;

void unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
{ … }
EXPORT_SYMBOL(…);

bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
{ … }
EXPORT_SYMBOL(…);

/* Select new prio value from the range, managed by kernel. */

static inline u32 tcf_auto_prio(struct tcf_proto *tp)
{ … }

static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
{ … }

static bool tcf_proto_is_unlocked(const char *kind)
{ … }

static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
					  u32 prio, struct tcf_chain *chain,
					  bool rtnl_held,
					  struct netlink_ext_ack *extack)
{ … }

static void tcf_proto_get(struct tcf_proto *tp)
{ … }

static void tcf_maintain_bypass(struct tcf_block *block)
{ … }

static void tcf_block_filter_cnt_update(struct tcf_block *block, bool *counted, bool add)
{ … }

static void tcf_chain_put(struct tcf_chain *chain);

static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
			      bool sig_destroy, struct netlink_ext_ack *extack)
{ … }

static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
			  struct netlink_ext_ack *extack)
{ … }

static bool tcf_proto_check_delete(struct tcf_proto *tp)
{ … }

static void tcf_proto_mark_delete(struct tcf_proto *tp)
{ … }

static bool tcf_proto_is_deleting(struct tcf_proto *tp)
{ … }

#define ASSERT_BLOCK_LOCKED(block) …

struct tcf_filter_chain_list_item { … };

static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
					  u32 chain_index)
{ … }

static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
				       struct tcf_proto *tp_head)
{ … }

static void tcf_chain0_head_change(struct tcf_chain *chain,
				   struct tcf_proto *tp_head)
{ … }

/* Returns true if block can be safely freed. */

static bool tcf_chain_detach(struct tcf_chain *chain)
{ … }

static void tcf_block_destroy(struct tcf_block *block)
{ … }

static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
{ … }

static void tcf_chain_hold(struct tcf_chain *chain)
{ … }

static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
{ … }

static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
					  u32 chain_index)
{ … }

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
					      u32 chain_index)
{ … }
#endif

static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
			   u32 seq, u16 flags, int event, bool unicast,
			   struct netlink_ext_ack *extack);

static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
					 u32 chain_index, bool create,
					 bool by_act)
{ … }

static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
				       bool create)
{ … }

struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
{ … }
EXPORT_SYMBOL(…);

static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
			       void *tmplt_priv);
static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
				  void *tmplt_priv, u32 chain_index,
				  struct tcf_block *block, struct sk_buff *oskb,
				  u32 seq, u16 flags);

static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
			    bool explicitly_created)
{ … }

static void tcf_chain_put(struct tcf_chain *chain)
{ … }

void tcf_chain_put_by_act(struct tcf_chain *chain)
{ … }
EXPORT_SYMBOL(…);

static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
{ … }

static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
{ … }

static int tcf_block_setup(struct tcf_block *block,
			   struct flow_block_offload *bo);

static void tcf_block_offload_init(struct flow_block_offload *bo,
				   struct net_device *dev, struct Qdisc *sch,
				   enum flow_block_command command,
				   enum flow_block_binder_type binder_type,
				   struct flow_block *flow_block,
				   bool shared, struct netlink_ext_ack *extack)
{ … }

static void tcf_block_unbind(struct tcf_block *block,
			     struct flow_block_offload *bo);

static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
{ … }

static bool tcf_block_offload_in_use(struct tcf_block *block)
{ … }

static int tcf_block_offload_cmd(struct tcf_block *block,
				 struct net_device *dev, struct Qdisc *sch,
				 struct tcf_block_ext_info *ei,
				 enum flow_block_command command,
				 struct netlink_ext_ack *extack)
{ … }

static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
				  struct tcf_block_ext_info *ei,
				  struct netlink_ext_ack *extack)
{ … }

static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
				     struct tcf_block_ext_info *ei)
{ … }

static int
tcf_chain0_head_change_cb_add(struct tcf_block *block,
			      struct tcf_block_ext_info *ei,
			      struct netlink_ext_ack *extack)
{ … }

static void
tcf_chain0_head_change_cb_del(struct tcf_block *block,
			      struct tcf_block_ext_info *ei)
{ … }

struct tcf_net { … };

static unsigned int tcf_net_id;

static int tcf_block_insert(struct tcf_block *block, struct net *net,
			    struct netlink_ext_ack *extack)
{ … }

static void tcf_block_remove(struct tcf_block *block, struct net *net)
{ … }

static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
					  u32 block_index,
					  struct netlink_ext_ack *extack)
{ … }

struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
{ … }
EXPORT_SYMBOL(…);

static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
{ … }

static struct tcf_chain *
__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
{ … }

/* Function to be used by all clients that want to iterate over all chains on
 * block. It properly obtains block->lock and takes reference to chain before
 * returning it. Users of this function must be tolerant to concurrent chain
 * insertion/deletion or ensure that no concurrent chain modification is
 * possible. Note that all netlink dump callbacks cannot guarantee to provide
 * consistent dump because rtnl lock is released each time skb is filled with
 * data and sent to user-space.
 */

struct tcf_chain *
tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
{ … }
EXPORT_SYMBOL(…);

static struct tcf_proto *
__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
{ … }

/* Function to be used by all clients that want to iterate over all tp's on
 * chain. Users of this function must be tolerant to concurrent tp
 * insertion/deletion or ensure that no concurrent chain modification is
 * possible. Note that all netlink dump callbacks cannot guarantee to provide
 * consistent dump because rtnl lock is released each time skb is filled with
 * data and sent to user-space.
 */

struct tcf_proto *
tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
{ … }
EXPORT_SYMBOL(…);

static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
{ … }

/* Lookup Qdisc and increments its reference counter.
 * Set parent, if necessary.
 */

static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
			    u32 *parent, int ifindex, bool rtnl_held,
			    struct netlink_ext_ack *extack)
{ … }

static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
			       int ifindex, struct netlink_ext_ack *extack)
{ … }

static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
					  unsigned long cl, int ifindex,
					  u32 block_index,
					  struct netlink_ext_ack *extack)
{ … }

static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
			    struct tcf_block_ext_info *ei, bool rtnl_held)
{ … }

static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
{ … }

/* Find tcf block.
 * Set q, parent, cl when appropriate.
 */

static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
					u32 *parent, unsigned long *cl,
					int ifindex, u32 block_index,
					struct netlink_ext_ack *extack)
{ … }

static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
			      bool rtnl_held)
{ … }

struct tcf_block_owner_item { … };

static void
tcf_block_owner_netif_keep_dst(struct tcf_block *block,
			       struct Qdisc *q,
			       enum flow_block_binder_type binder_type)
{ … }

void tcf_block_netif_keep_dst(struct tcf_block *block)
{ … }
EXPORT_SYMBOL(…);

static int tcf_block_owner_add(struct tcf_block *block,
			       struct Qdisc *q,
			       enum flow_block_binder_type binder_type)
{ … }

static void tcf_block_owner_del(struct tcf_block *block,
				struct Qdisc *q,
				enum flow_block_binder_type binder_type)
{ … }

static bool tcf_block_tracks_dev(struct tcf_block *block,
				 struct tcf_block_ext_info *ei)
{ … }

int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
		      struct tcf_block_ext_info *ei,
		      struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
{ … }

int tcf_block_get(struct tcf_block **p_block,
		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
		  struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

/* XXX: Standalone actions are not allowed to jump to any chain, and bound
 * actions should be all removed after flushing.
 */
void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
		       struct tcf_block_ext_info *ei)
{ … }
EXPORT_SYMBOL(…);

void tcf_block_put(struct tcf_block *block)
{ … }

EXPORT_SYMBOL(…);

static int
tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
			    void *cb_priv, bool add, bool offload_in_use,
			    struct netlink_ext_ack *extack)
{ … }

static int tcf_block_bind(struct tcf_block *block,
			  struct flow_block_offload *bo)
{ … }

static void tcf_block_unbind(struct tcf_block *block,
			     struct flow_block_offload *bo)
{ … }

static int tcf_block_setup(struct tcf_block *block,
			   struct flow_block_offload *bo)
{ … }

/* Main classifier routine: scans classifier chain attached
 * to this qdisc, (optionally) tests for protocol and asks
 * specific classifiers.
 */
static inline int __tcf_classify(struct sk_buff *skb,
				 const struct tcf_proto *tp,
				 const struct tcf_proto *orig_tp,
				 struct tcf_result *res,
				 bool compat_mode,
				 struct tcf_exts_miss_cookie_node *n,
				 int act_index,
				 u32 *last_executed_chain)
{ … }

int tcf_classify(struct sk_buff *skb,
		 const struct tcf_block *block,
		 const struct tcf_proto *tp,
		 struct tcf_result *res, bool compat_mode)
{ … }
EXPORT_SYMBOL(…);

struct tcf_chain_info { … };

static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
					   struct tcf_chain_info *chain_info)
{ … }

static int tcf_chain_tp_insert(struct tcf_chain *chain,
			       struct tcf_chain_info *chain_info,
			       struct tcf_proto *tp)
{ … }

static void tcf_chain_tp_remove(struct tcf_chain *chain,
				struct tcf_chain_info *chain_info,
				struct tcf_proto *tp)
{ … }

static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
					   struct tcf_chain_info *chain_info,
					   u32 protocol, u32 prio,
					   bool prio_allocate);

/* Try to insert new proto.
 * If proto with specified priority already exists, free new proto
 * and return existing one.
 */

static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
						    struct tcf_proto *tp_new,
						    u32 protocol, u32 prio,
						    bool rtnl_held)
{ … }

static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
				      struct tcf_proto *tp, bool rtnl_held,
				      struct netlink_ext_ack *extack)
{ … }

static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
					   struct tcf_chain_info *chain_info,
					   u32 protocol, u32 prio,
					   bool prio_allocate)
{ … }

static int tcf_fill_node(struct net *net, struct sk_buff *skb,
			 struct tcf_proto *tp, struct tcf_block *block,
			 struct Qdisc *q, u32 parent, void *fh,
			 u32 portid, u32 seq, u16 flags, int event,
			 bool terse_dump, bool rtnl_held,
			 struct netlink_ext_ack *extack)
{ … }

static int tfilter_notify(struct net *net, struct sk_buff *oskb,
			  struct nlmsghdr *n, struct tcf_proto *tp,
			  struct tcf_block *block, struct Qdisc *q,
			  u32 parent, void *fh, int event, bool unicast,
			  bool rtnl_held, struct netlink_ext_ack *extack)
{ … }

static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
			      struct nlmsghdr *n, struct tcf_proto *tp,
			      struct tcf_block *block, struct Qdisc *q,
			      u32 parent, void *fh, bool *last, bool rtnl_held,
			      struct netlink_ext_ack *extack)
{ … }

static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
				 struct tcf_block *block, struct Qdisc *q,
				 u32 parent, struct nlmsghdr *n,
				 struct tcf_chain *chain, int event,
				 struct netlink_ext_ack *extack)
{ … }

static void tfilter_put(struct tcf_proto *tp, void *fh)
{ … }

static bool is_qdisc_ingress(__u32 classid)
{ … }

static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
			  struct netlink_ext_ack *extack)
{ … }

static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
			  struct netlink_ext_ack *extack)
{ … }

static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
			  struct netlink_ext_ack *extack)
{ … }

struct tcf_dump_args { … };

static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
{ … }

static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
			   struct sk_buff *skb, struct netlink_callback *cb,
			   long index_start, long *p_index, bool terse)
{ … }

static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = …;

/* called with RTNL */
static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
{ … }

static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
			      void *tmplt_priv, u32 chain_index,
			      struct net *net, struct sk_buff *skb,
			      struct tcf_block *block,
			      u32 portid, u32 seq, u16 flags, int event,
			      struct netlink_ext_ack *extack)
{ … }

static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
			   u32 seq, u16 flags, int event, bool unicast,
			   struct netlink_ext_ack *extack)
{ … }

static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
				  void *tmplt_priv, u32 chain_index,
				  struct tcf_block *block, struct sk_buff *oskb,
				  u32 seq, u16 flags)
{ … }

static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
			      struct nlattr **tca,
			      struct netlink_ext_ack *extack)
{ … }

static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
			       void *tmplt_priv)
{ … }

/* Add/delete/get a chain */

static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
			struct netlink_ext_ack *extack)
{ … }

/* called with RTNL */
static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
{ … }

int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
		     int police, struct tcf_proto *tp, u32 handle,
		     bool use_action_miss)
{ … }
EXPORT_SYMBOL(…);

void tcf_exts_destroy(struct tcf_exts *exts)
{ … }
EXPORT_SYMBOL(…);

int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
			 struct nlattr *rate_tlv, struct tcf_exts *exts,
			 u32 flags, u32 fl_flags, struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
		      struct nlattr *rate_tlv, struct tcf_exts *exts,
		      u32 flags, struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
{ … }
EXPORT_SYMBOL(…);

#ifdef CONFIG_NET_CLS_ACT
static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
{ … }
#endif

int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
{ … }
EXPORT_SYMBOL(…);

int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
{ … }
EXPORT_SYMBOL(…);

int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
{ … }
EXPORT_SYMBOL(…);

static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
{ … }

static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
{ … }

static void tc_cls_offload_cnt_update(struct tcf_block *block,
				      struct tcf_proto *tp, u32 *cnt,
				      u32 *flags, u32 diff, bool add)
{ … }

static void
tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
			 u32 *cnt, u32 *flags)
{ … }

static int
__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
		   void *type_data, bool err_stop)
{ … }

int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
		     void *type_data, bool err_stop, bool rtnl_held)
{ … }
EXPORT_SYMBOL(…);

/* Non-destructive filter add. If filter that wasn't already in hardware is
 * successfully offloaded, increment block offloads counter. On failure,
 * previously offloaded filter is considered to be intact and offloads counter
 * is not decremented.
 */

int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
		    enum tc_setup_type type, void *type_data, bool err_stop,
		    u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
{ … }
EXPORT_SYMBOL(…);

/* Destructive filter replace. If filter that wasn't already in hardware is
 * successfully offloaded, increment block offload counter. On failure,
 * previously offloaded filter is considered to be destroyed and offload counter
 * is decremented.
 */

int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
			enum tc_setup_type type, void *type_data, bool err_stop,
			u32 *old_flags, unsigned int *old_in_hw_count,
			u32 *new_flags, unsigned int *new_in_hw_count,
			bool rtnl_held)
{ … }
EXPORT_SYMBOL(…);

/* Destroy filter and decrement block offload counter, if filter was previously
 * offloaded.
 */

int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
			enum tc_setup_type type, void *type_data, bool err_stop,
			u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
{ … }
EXPORT_SYMBOL(…);

int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
			  bool add, flow_setup_cb_t *cb,
			  enum tc_setup_type type, void *type_data,
			  void *cb_priv, u32 *flags, unsigned int *in_hw_count)
{ … }
EXPORT_SYMBOL(…);

static int tcf_act_get_user_cookie(struct flow_action_entry *entry,
				   const struct tc_action *act)
{ … }

static void tcf_act_put_user_cookie(struct flow_action_entry *entry)
{ … }

void tc_cleanup_offload_action(struct flow_action *flow_action)
{ … }
EXPORT_SYMBOL(…);

static int tc_setup_offload_act(struct tc_action *act,
				struct flow_action_entry *entry,
				u32 *index_inc,
				struct netlink_ext_ack *extack)
{ … }

int tc_setup_action(struct flow_action *flow_action,
		    struct tc_action *actions[],
		    u32 miss_cookie_base,
		    struct netlink_ext_ack *extack)
{ … }

int tc_setup_offload_action(struct flow_action *flow_action,
			    const struct tcf_exts *exts,
			    struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
{ … }
EXPORT_SYMBOL(…);

#ifdef CONFIG_NET_CLS_ACT
static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
					u32 *p_block_index,
					struct netlink_ext_ack *extack)
{ … }

int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
		    enum flow_block_binder_type binder_type,
		    struct nlattr *block_index_attr,
		    struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
{ … }
EXPORT_SYMBOL(…);

int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
			       struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
				  struct sk_buff **to_free, int *ret)
{ … }
EXPORT_SYMBOL(…);

int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
{ … }
EXPORT_SYMBOL(…);
#endif

static __net_init int tcf_net_init(struct net *net)
{ … }

static void __net_exit tcf_net_exit(struct net *net)
{ … }

static struct pernet_operations tcf_net_ops = …;

static int __init tc_filter_init(void)
{ … }

subsys_initcall(tc_filter_init);