linux/net/sched/act_api.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/act_api.c	Packet action API.
 *
 * Author:	Jamal Hadi Salim
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/err.h>
#include <linux/module.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <net/sch_generic.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_pedit.h>
#include <net/act_api.h>
#include <net/netlink.h>
#include <net/flow_offload.h>
#include <net/tc_wrapper.h>

#ifdef CONFIG_INET
DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count);
EXPORT_SYMBOL_GPL(…);
#endif

int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb))
{ … }
EXPORT_SYMBOL_GPL(…);

static void tcf_action_goto_chain_exec(const struct tc_action *a,
				       struct tcf_result *res)
{ … }

static void tcf_free_cookie_rcu(struct rcu_head *p)
{ … }

static void tcf_set_action_cookie(struct tc_cookie __rcu **old_cookie,
				  struct tc_cookie *new_cookie)
{ … }

int tcf_action_check_ctrlact(int action, struct tcf_proto *tp,
			     struct tcf_chain **newchain,
			     struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action,
					 struct tcf_chain *goto_chain)
{ … }
EXPORT_SYMBOL(…);

/* XXX: For standalone actions, we don't need a RCU grace period either, because
 * actions are always connected to filters and filters are already destroyed in
 * RCU callbacks, so after a RCU grace period actions are already disconnected
 * from filters. Readers later can not find us.
 */
static void free_tcf(struct tc_action *p)
{ … }

static void offload_action_hw_count_set(struct tc_action *act,
					u32 hw_count)
{ … }

static void offload_action_hw_count_inc(struct tc_action *act,
					u32 hw_count)
{ … }

static void offload_action_hw_count_dec(struct tc_action *act,
					u32 hw_count)
{ … }

static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act)
{ … }

static bool tc_act_skip_hw(u32 flags)
{ … }

static bool tc_act_skip_sw(u32 flags)
{ … }

/* SKIP_HW and SKIP_SW are mutually exclusive flags. */
static bool tc_act_flags_valid(u32 flags)
{ … }

static int offload_action_init(struct flow_offload_action *fl_action,
			       struct tc_action *act,
			       enum offload_act_command  cmd,
			       struct netlink_ext_ack *extack)
{ … }

static int tcf_action_offload_cmd_ex(struct flow_offload_action *fl_act,
				     u32 *hw_count)
{ … }

static int tcf_action_offload_cmd_cb_ex(struct flow_offload_action *fl_act,
					u32 *hw_count,
					flow_indr_block_bind_cb_t *cb,
					void *cb_priv)
{ … }

static int tcf_action_offload_cmd(struct flow_offload_action *fl_act,
				  u32 *hw_count,
				  flow_indr_block_bind_cb_t *cb,
				  void *cb_priv)
{ … }

static int tcf_action_offload_add_ex(struct tc_action *action,
				     struct netlink_ext_ack *extack,
				     flow_indr_block_bind_cb_t *cb,
				     void *cb_priv)
{ … }

/* offload the tc action after it is inserted */
static int tcf_action_offload_add(struct tc_action *action,
				  struct netlink_ext_ack *extack)
{ … }

int tcf_action_update_hw_stats(struct tc_action *action)
{ … }
EXPORT_SYMBOL(…);

static int tcf_action_offload_del_ex(struct tc_action *action,
				     flow_indr_block_bind_cb_t *cb,
				     void *cb_priv)
{ … }

static int tcf_action_offload_del(struct tc_action *action)
{ … }

static void tcf_action_cleanup(struct tc_action *p)
{ … }

static int __tcf_action_put(struct tc_action *p, bool bind)
{ … }

static int __tcf_idr_release(struct tc_action *p, bool bind, bool strict)
{ … }

int tcf_idr_release(struct tc_action *a, bool bind)
{ … }
EXPORT_SYMBOL(…);

static size_t tcf_action_shared_attrs_size(const struct tc_action *act)
{ … }

static size_t tcf_action_full_attrs_size(size_t sz)
{ … }

static size_t tcf_action_fill_size(const struct tc_action *act)
{ … }

static int
tcf_action_dump_terse(struct sk_buff *skb, struct tc_action *a, bool from_act)
{ … }

static int tcf_dump_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb,
			   struct netlink_callback *cb)
{ … }

static int tcf_idr_release_unsafe(struct tc_action *p)
{ … }

static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb,
			  const struct tc_action_ops *ops,
			  struct netlink_ext_ack *extack)
{ … }

int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb,
		       struct netlink_callback *cb, int type,
		       const struct tc_action_ops *ops,
		       struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index)
{ … }
EXPORT_SYMBOL(…);

static int __tcf_generic_walker(struct net *net, struct sk_buff *skb,
				struct netlink_callback *cb, int type,
				const struct tc_action_ops *ops,
				struct netlink_ext_ack *extack)
{ … }

static int __tcf_idr_search(struct net *net,
			    const struct tc_action_ops *ops,
			    struct tc_action **a, u32 index)
{ … }

static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index)
{ … }

int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est,
		   struct tc_action **a, const struct tc_action_ops *ops,
		   int bind, bool cpustats, u32 flags)
{ … }
EXPORT_SYMBOL(…);

int tcf_idr_create_from_flags(struct tc_action_net *tn, u32 index,
			      struct nlattr *est, struct tc_action **a,
			      const struct tc_action_ops *ops, int bind,
			      u32 flags)
{ … }
EXPORT_SYMBOL(…);

/* Cleanup idr index that was allocated but not initialized. */

void tcf_idr_cleanup(struct tc_action_net *tn, u32 index)
{ … }
EXPORT_SYMBOL(…);

/* Check if action with specified index exists. If actions is found, increments
 * its reference and bind counters, and return 1. Otherwise insert temporary
 * error pointer (to prevent concurrent users from inserting actions with same
 * index) and return 0.
 *
 * May return -EAGAIN for binding actions in case of a parallel add/delete on
 * the requested index.
 */

int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index,
			struct tc_action **a, int bind)
{ … }
EXPORT_SYMBOL(…);

void tcf_idrinfo_destroy(const struct tc_action_ops *ops,
			 struct tcf_idrinfo *idrinfo)
{ … }
EXPORT_SYMBOL(…);

static LIST_HEAD(act_base);
static DEFINE_RWLOCK(act_mod_lock);
/* since act ops id is stored in pernet subsystem list,
 * then there is no way to walk through only all the action
 * subsystem, so we keep tc action pernet ops id for
 * reoffload to walk through.
 */
static LIST_HEAD(act_pernet_id_list);
static DEFINE_MUTEX(act_id_mutex);
struct tc_act_pernet_id { … };

static int tcf_pernet_add_id_list(unsigned int id)
{ … }

static void tcf_pernet_del_id_list(unsigned int id)
{ … }

int tcf_register_action(struct tc_action_ops *act,
			struct pernet_operations *ops)
{ … }
EXPORT_SYMBOL(…);

int tcf_unregister_action(struct tc_action_ops *act,
			  struct pernet_operations *ops)
{ … }
EXPORT_SYMBOL(…);

/* lookup by name */
static struct tc_action_ops *tc_lookup_action_n(char *kind)
{ … }

/* lookup by nlattr */
static struct tc_action_ops *tc_lookup_action(struct nlattr *kind)
{ … }

/*TCA_ACT_MAX_PRIO is 32, there count up to 32 */
#define TCA_ACT_MAX_PRIO_MASK …
int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions,
		    int nr_actions, struct tcf_result *res)
{ … }
EXPORT_SYMBOL(…);

int tcf_action_destroy(struct tc_action *actions[], int bind)
{ … }

static int tcf_action_put(struct tc_action *p)
{ … }

static void tcf_action_put_many(struct tc_action *actions[])
{ … }

static void tca_put_bound_many(struct tc_action *actions[], int init_res[])
{ … }

int
tcf_action_dump_old(struct sk_buff *skb, struct tc_action *a, int bind, int ref)
{ … }

int
tcf_action_dump_1(struct sk_buff *skb, struct tc_action *a, int bind, int ref)
{ … }
EXPORT_SYMBOL(…);

int tcf_action_dump(struct sk_buff *skb, struct tc_action *actions[],
		    int bind, int ref, bool terse)
{ … }

static struct tc_cookie *nla_memdup_cookie(struct nlattr **tb)
{ … }

static u8 tcf_action_hw_stats_get(struct nlattr *hw_stats_attr)
{ … }

static const struct nla_policy tcf_action_policy[TCA_ACT_MAX + 1] = …;

void tcf_idr_insert_many(struct tc_action *actions[], int init_res[])
{ … }

struct tc_action_ops *tc_action_load_ops(struct nlattr *nla, u32 flags,
					 struct netlink_ext_ack *extack)
{ … }

struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp,
				    struct nlattr *nla, struct nlattr *est,
				    struct tc_action_ops *a_o, int *init_res,
				    u32 flags, struct netlink_ext_ack *extack)
{ … }

static bool tc_act_bind(u32 flags)
{ … }

/* Returns numbers of initialized actions or negative error. */

int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla,
		    struct nlattr *est, struct tc_action *actions[],
		    int init_res[], size_t *attr_size,
		    u32 flags, u32 fl_flags,
		    struct netlink_ext_ack *extack)
{ … }

void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets,
			     u64 drops, bool hw)
{ … }
EXPORT_SYMBOL(…);

int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *p,
			  int compat_mode)
{ … }

static int tca_get_fill(struct sk_buff *skb, struct tc_action *actions[],
			u32 portid, u32 seq, u16 flags, int event, int bind,
			int ref, struct netlink_ext_ack *extack)
{ … }

static int
tcf_get_notify(struct net *net, u32 portid, struct nlmsghdr *n,
	       struct tc_action *actions[], int event,
	       struct netlink_ext_ack *extack)
{ … }

static struct tc_action *tcf_action_get_1(struct net *net, struct nlattr *nla,
					  struct nlmsghdr *n, u32 portid,
					  struct netlink_ext_ack *extack)
{ … }

static int tca_action_flush(struct net *net, struct nlattr *nla,
			    struct nlmsghdr *n, u32 portid,
			    struct netlink_ext_ack *extack)
{ … }

static int tcf_action_delete(struct net *net, struct tc_action *actions[])
{ … }

static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net,
						    struct tc_action *action)
{ … }

static int tcf_reoffload_del_notify(struct net *net, struct tc_action *action)
{ … }

int tcf_action_reoffload_cb(flow_indr_block_bind_cb_t *cb,
			    void *cb_priv, bool add)
{ … }

static struct sk_buff *tcf_del_notify_msg(struct net *net, struct nlmsghdr *n,
					  struct tc_action *actions[],
					  u32 portid, size_t attr_size,
					  struct netlink_ext_ack *extack)
{ … }

static int tcf_del_notify(struct net *net, struct nlmsghdr *n,
			  struct tc_action *actions[], u32 portid,
			  size_t attr_size, struct netlink_ext_ack *extack)
{ … }

static int
tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n,
	      u32 portid, int event, struct netlink_ext_ack *extack)
{ … }

static struct sk_buff *tcf_add_notify_msg(struct net *net, struct nlmsghdr *n,
					  struct tc_action *actions[],
					  u32 portid, size_t attr_size,
					  struct netlink_ext_ack *extack)
{ … }

static int tcf_add_notify(struct net *net, struct nlmsghdr *n,
			  struct tc_action *actions[], u32 portid,
			  size_t attr_size, struct netlink_ext_ack *extack)
{ … }

static int tcf_action_add(struct net *net, struct nlattr *nla,
			  struct nlmsghdr *n, u32 portid, u32 flags,
			  struct netlink_ext_ack *extack)
{ … }

static const struct nla_policy tcaa_policy[TCA_ROOT_MAX + 1] = …;

static int tc_ctl_action(struct sk_buff *skb, struct nlmsghdr *n,
			 struct netlink_ext_ack *extack)
{ … }

static struct nlattr *find_dump_kind(struct nlattr **nla)
{ … }

static int tc_dump_action(struct sk_buff *skb, struct netlink_callback *cb)
{ … }

static int __init tc_action_init(void)
{ … }

subsys_initcall(tc_action_init);