linux/net/ipv4/nexthop.c

// SPDX-License-Identifier: GPL-2.0
/* Generic nexthop implementation
 *
 * Copyright (c) 2017-19 Cumulus Networks
 * Copyright (c) 2017-19 David Ahern <[email protected]>
 */

#include <linux/nexthop.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <net/arp.h>
#include <net/ipv6_stubs.h>
#include <net/lwtunnel.h>
#include <net/ndisc.h>
#include <net/nexthop.h>
#include <net/route.h>
#include <net/sock.h>

#define NH_RES_DEFAULT_IDLE_TIMER …
#define NH_RES_DEFAULT_UNBALANCED_TIMER …

static void remove_nexthop(struct net *net, struct nexthop *nh,
			   struct nl_info *nlinfo);

#define NH_DEV_HASHBITS …
#define NH_DEV_HASHSIZE …

#define NHA_OP_FLAGS_DUMP_ALL …

static const struct nla_policy rtm_nh_policy_new[] = …;

static const struct nla_policy rtm_nh_policy_get[] = …;

static const struct nla_policy rtm_nh_policy_del[] = …;

static const struct nla_policy rtm_nh_policy_dump[] = …;

static const struct nla_policy rtm_nh_res_policy_new[] = …;

static const struct nla_policy rtm_nh_policy_dump_bucket[] = …;

static const struct nla_policy rtm_nh_res_bucket_policy_dump[] = …;

static const struct nla_policy rtm_nh_policy_get_bucket[] = …;

static const struct nla_policy rtm_nh_res_bucket_policy_get[] = …;

static bool nexthop_notifiers_is_empty(struct net *net)
{ … }

static void
__nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
			       const struct nh_info *nhi)
{ … }

static int nh_notifier_single_info_init(struct nh_notifier_info *info,
					const struct nexthop *nh)
{ … }

static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
{ … }

static int nh_notifier_mpath_info_init(struct nh_notifier_info *info,
				       struct nh_group *nhg)
{ … }

static int nh_notifier_res_table_info_init(struct nh_notifier_info *info,
					   struct nh_group *nhg)
{ … }

static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
				     const struct nexthop *nh)
{ … }

static void nh_notifier_grp_info_fini(struct nh_notifier_info *info,
				      const struct nexthop *nh)
{ … }

static int nh_notifier_info_init(struct nh_notifier_info *info,
				 const struct nexthop *nh)
{ … }

static void nh_notifier_info_fini(struct nh_notifier_info *info,
				  const struct nexthop *nh)
{ … }

static int call_nexthop_notifiers(struct net *net,
				  enum nexthop_event_type event_type,
				  struct nexthop *nh,
				  struct netlink_ext_ack *extack)
{ … }

static int
nh_notifier_res_bucket_idle_timer_get(const struct nh_notifier_info *info,
				      bool force, unsigned int *p_idle_timer_ms)
{ … }

static int nh_notifier_res_bucket_info_init(struct nh_notifier_info *info,
					    u16 bucket_index, bool force,
					    struct nh_info *oldi,
					    struct nh_info *newi)
{ … }

static void nh_notifier_res_bucket_info_fini(struct nh_notifier_info *info)
{ … }

static int __call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
					       u16 bucket_index, bool force,
					       struct nh_info *oldi,
					       struct nh_info *newi,
					       struct netlink_ext_ack *extack)
{ … }

/* There are three users of RES_TABLE, and NHs etc. referenced from there:
 *
 * 1) a collection of callbacks for NH maintenance. This operates under
 *    RTNL,
 * 2) the delayed work that gradually balances the resilient table,
 * 3) and nexthop_select_path(), operating under RCU.
 *
 * Both the delayed work and the RTNL block are writers, and need to
 * maintain mutual exclusion. Since there are only two and well-known
 * writers for each table, the RTNL code can make sure it has exclusive
 * access thus:
 *
 * - Have the DW operate without locking;
 * - synchronously cancel the DW;
 * - do the writing;
 * - if the write was not actually a delete, call upkeep, which schedules
 *   DW again if necessary.
 *
 * The functions that are always called from the RTNL context use
 * rtnl_dereference(). The functions that can also be called from the DW do
 * a raw dereference and rely on the above mutual exclusion scheme.
 */
#define nh_res_dereference(p) …

static int call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
					     u16 bucket_index, bool force,
					     struct nexthop *old_nh,
					     struct nexthop *new_nh,
					     struct netlink_ext_ack *extack)
{ … }

static int call_nexthop_res_table_notifiers(struct net *net, struct nexthop *nh,
					    struct netlink_ext_ack *extack)
{ … }

static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
				 enum nexthop_event_type event_type,
				 struct nexthop *nh,
				 struct netlink_ext_ack *extack)
{ … }

static unsigned int nh_dev_hashfn(unsigned int val)
{ … }

static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
{ … }

static void nexthop_free_group(struct nexthop *nh)
{ … }

static void nexthop_free_single(struct nexthop *nh)
{ … }

void nexthop_free_rcu(struct rcu_head *head)
{ … }
EXPORT_SYMBOL_GPL(…);

static struct nexthop *nexthop_alloc(void)
{ … }

static struct nh_group *nexthop_grp_alloc(u16 num_nh)
{ … }

static void nh_res_table_upkeep_dw(struct work_struct *work);

static struct nh_res_table *
nexthop_res_table_alloc(struct net *net, u32 nhg_id, struct nh_config *cfg)
{ … }

static void nh_base_seq_inc(struct net *net)
{ … }

/* no reference taken; rcu lock or rtnl must be held */
struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
{ … }
EXPORT_SYMBOL_GPL(…);

/* used for auto id allocation; called with rtnl held */
static u32 nh_find_unused_id(struct net *net)
{ … }

static void nh_res_time_set_deadline(unsigned long next_time,
				     unsigned long *deadline)
{ … }

static clock_t nh_res_table_unbalanced_time(struct nh_res_table *res_table)
{ … }

static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)
{ … }

static void nh_grp_entry_stats_inc(struct nh_grp_entry *nhge)
{ … }

static void nh_grp_entry_stats_read(struct nh_grp_entry *nhge,
				    u64 *ret_packets)
{ … }

static int nh_notifier_grp_hw_stats_init(struct nh_notifier_info *info,
					 const struct nexthop *nh)
{ … }

static void nh_notifier_grp_hw_stats_fini(struct nh_notifier_info *info)
{ … }

void nh_grp_hw_stats_report_delta(struct nh_notifier_grp_hw_stats_info *info,
				  unsigned int nh_idx,
				  u64 delta_packets)
{ … }
EXPORT_SYMBOL(…);

static void nh_grp_hw_stats_apply_update(struct nexthop *nh,
					 struct nh_notifier_info *info)
{ … }

static int nh_grp_hw_stats_update(struct nexthop *nh, bool *hw_stats_used)
{ … }

static int nla_put_nh_group_stats_entry(struct sk_buff *skb,
					struct nh_grp_entry *nhge,
					u32 op_flags)
{ … }

static int nla_put_nh_group_stats(struct sk_buff *skb, struct nexthop *nh,
				  u32 op_flags)
{ … }

static int nla_put_nh_group(struct sk_buff *skb, struct nexthop *nh,
			    u32 op_flags)
{ … }

static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
			int event, u32 portid, u32 seq, unsigned int nlflags,
			u32 op_flags)
{ … }

static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)
{ … }

static size_t nh_nlmsg_size_grp(struct nexthop *nh)
{ … }

static size_t nh_nlmsg_size_single(struct nexthop *nh)
{ … }

static size_t nh_nlmsg_size(struct nexthop *nh)
{ … }

static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
{ … }

static unsigned long nh_res_bucket_used_time(const struct nh_res_bucket *bucket)
{ … }

static unsigned long
nh_res_bucket_idle_point(const struct nh_res_table *res_table,
			 const struct nh_res_bucket *bucket,
			 unsigned long now)
{ … }

static unsigned long
nh_res_table_unb_point(const struct nh_res_table *res_table)
{ … }

static void nh_res_bucket_set_idle(const struct nh_res_table *res_table,
				   struct nh_res_bucket *bucket)
{ … }

static void nh_res_bucket_set_busy(struct nh_res_bucket *bucket)
{ … }

static clock_t nh_res_bucket_idle_time(const struct nh_res_bucket *bucket)
{ … }

static int nh_fill_res_bucket(struct sk_buff *skb, struct nexthop *nh,
			      struct nh_res_bucket *bucket, u16 bucket_index,
			      int event, u32 portid, u32 seq,
			      unsigned int nlflags,
			      struct netlink_ext_ack *extack)
{ … }

static void nexthop_bucket_notify(struct nh_res_table *res_table,
				  u16 bucket_index)
{ … }

static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
			   bool *is_fdb, struct netlink_ext_ack *extack)
{ … }

static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
				   struct netlink_ext_ack *extack)
{ … }

static int nh_check_attr_group(struct net *net,
			       struct nlattr *tb[], size_t tb_size,
			       u16 nh_grp_type, struct netlink_ext_ack *extack)
{ … }

static bool ipv6_good_nh(const struct fib6_nh *nh)
{ … }

static bool ipv4_good_nh(const struct fib_nh *nh)
{ … }

static bool nexthop_is_good_nh(const struct nexthop *nh)
{ … }

static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)
{ … }

static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)
{ … }

static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)
{ … }

struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
{ … }
EXPORT_SYMBOL_GPL(…);

int nexthop_for_each_fib6_nh(struct nexthop *nh,
			     int (*cb)(struct fib6_nh *nh, void *arg),
			     void *arg)
{ … }
EXPORT_SYMBOL_GPL(…);

static int check_src_addr(const struct in6_addr *saddr,
			  struct netlink_ext_ack *extack)
{ … }

int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
		       struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL_GPL(…);

/* if existing nexthop has ipv6 routes linked to it, need
 * to verify this new spec works with ipv6
 */
static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
			      struct netlink_ext_ack *extack)
{ … }

static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
			       struct netlink_ext_ack *extack)
{ … }

/* Invoked by fib add code to verify nexthop by id is ok with
 * config for prefix; parts of fib_check_nh not done when nexthop
 * object is used.
 */
int fib_check_nexthop(struct nexthop *nh, u8 scope,
		      struct netlink_ext_ack *extack)
{ … }

static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
			     struct netlink_ext_ack *extack)
{ … }

static bool nh_res_nhge_is_balanced(const struct nh_grp_entry *nhge)
{ … }

static bool nh_res_nhge_is_ow(const struct nh_grp_entry *nhge)
{ … }

static bool nh_res_nhge_is_uw(const struct nh_grp_entry *nhge)
{ … }

static bool nh_res_table_is_balanced(const struct nh_res_table *res_table)
{ … }

static void nh_res_bucket_unset_nh(struct nh_res_bucket *bucket)
{ … }

static void nh_res_bucket_set_nh(struct nh_res_bucket *bucket,
				 struct nh_grp_entry *nhge)
{ … }

static bool nh_res_bucket_should_migrate(struct nh_res_table *res_table,
					 struct nh_res_bucket *bucket,
					 unsigned long *deadline, bool *force)
{ … }

static bool nh_res_bucket_migrate(struct nh_res_table *res_table,
				  u16 bucket_index, bool notify,
				  bool notify_nl, bool force)
{ … }

#define NH_RES_UPKEEP_DW_MINIMUM_INTERVAL …

static void nh_res_table_upkeep(struct nh_res_table *res_table,
				bool notify, bool notify_nl)
{ … }

static void nh_res_table_upkeep_dw(struct work_struct *work)
{ … }

static void nh_res_table_cancel_upkeep(struct nh_res_table *res_table)
{ … }

static void nh_res_group_rebalance(struct nh_group *nhg,
				   struct nh_res_table *res_table)
{ … }

/* Migrate buckets in res_table so that they reference NHGE's from NHG with
 * the right NH ID. Set those buckets that do not have a corresponding NHGE
 * entry in NHG as not occupied.
 */
static void nh_res_table_migrate_buckets(struct nh_res_table *res_table,
					 struct nh_group *nhg)
{ … }

static void replace_nexthop_grp_res(struct nh_group *oldg,
				    struct nh_group *newg)
{ … }

static void nh_hthr_group_rebalance(struct nh_group *nhg)
{ … }

static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
				struct nl_info *nlinfo)
{ … }

static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
				       struct nl_info *nlinfo)
{ … }

static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
{ … }

/* not called for nexthop replace */
static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
{ … }

static void __remove_nexthop(struct net *net, struct nexthop *nh,
			     struct nl_info *nlinfo)
{ … }

static void remove_nexthop(struct net *net, struct nexthop *nh,
			   struct nl_info *nlinfo)
{ … }

/* if any FIB entries reference this nexthop, any dst entries
 * need to be regenerated
 */
static void nh_rt_cache_flush(struct net *net, struct nexthop *nh,
			      struct nexthop *replaced_nh)
{ … }

static int replace_nexthop_grp(struct net *net, struct nexthop *old,
			       struct nexthop *new, const struct nh_config *cfg,
			       struct netlink_ext_ack *extack)
{ … }

static void nh_group_v4_update(struct nh_group *nhg)
{ … }

static int replace_nexthop_single_notify_res(struct net *net,
					     struct nh_res_table *res_table,
					     struct nexthop *old,
					     struct nh_info *oldi,
					     struct nh_info *newi,
					     struct netlink_ext_ack *extack)
{ … }

static int replace_nexthop_single_notify(struct net *net,
					 struct nexthop *group_nh,
					 struct nexthop *old,
					 struct nh_info *oldi,
					 struct nh_info *newi,
					 struct netlink_ext_ack *extack)
{ … }

static int replace_nexthop_single(struct net *net, struct nexthop *old,
				  struct nexthop *new,
				  struct netlink_ext_ack *extack)
{ … }

static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
				     struct nl_info *info)
{ … }

/* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
 * linked to this nexthop and for all groups that the nexthop
 * is a member of
 */
static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
				   struct nl_info *info)
{ … }

static int replace_nexthop(struct net *net, struct nexthop *old,
			   struct nexthop *new, const struct nh_config *cfg,
			   struct netlink_ext_ack *extack)
{ … }

/* called with rtnl_lock held */
static int insert_nexthop(struct net *net, struct nexthop *new_nh,
			  struct nh_config *cfg, struct netlink_ext_ack *extack)
{ … }

/* rtnl */
/* remove all nexthops tied to a device being deleted */
static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
{ … }

/* rtnl; called when net namespace is deleted */
static void flush_all_nexthops(struct net *net)
{ … }

static struct nexthop *nexthop_create_group(struct net *net,
					    struct nh_config *cfg)
{ … }

static int nh_create_ipv4(struct net *net, struct nexthop *nh,
			  struct nh_info *nhi, struct nh_config *cfg,
			  struct netlink_ext_ack *extack)
{ … }

static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
			  struct nh_info *nhi, struct nh_config *cfg,
			  struct netlink_ext_ack *extack)
{ … }

static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
				      struct netlink_ext_ack *extack)
{ … }

/* called with rtnl lock held */
static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
				   struct netlink_ext_ack *extack)
{ … }

static int rtm_nh_get_timer(struct nlattr *attr, unsigned long fallback,
			    unsigned long *timer_p, bool *has_p,
			    struct netlink_ext_ack *extack)
{ … }

static int rtm_to_nh_config_grp_res(struct nlattr *res, struct nh_config *cfg,
				    struct netlink_ext_ack *extack)
{ … }

static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
			    struct nlmsghdr *nlh, struct nh_config *cfg,
			    struct netlink_ext_ack *extack)
{ … }

/* rtnl */
static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
			   struct netlink_ext_ack *extack)
{ … }

static int nh_valid_get_del_req(const struct nlmsghdr *nlh,
				struct nlattr **tb, u32 *id, u32 *op_flags,
				struct netlink_ext_ack *extack)
{ … }

/* rtnl */
static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
			   struct netlink_ext_ack *extack)
{ … }

/* rtnl */
static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
			   struct netlink_ext_ack *extack)
{ … }

struct nh_dump_filter { … };

static bool nh_dump_filtered(struct nexthop *nh,
			     struct nh_dump_filter *filter, u8 family)
{ … }

static int __nh_valid_dump_req(const struct nlmsghdr *nlh, struct nlattr **tb,
			       struct nh_dump_filter *filter,
			       struct netlink_ext_ack *extack)
{ … }

static int nh_valid_dump_req(const struct nlmsghdr *nlh,
			     struct nh_dump_filter *filter,
			     struct netlink_callback *cb)
{ … }

struct rtm_dump_nh_ctx { … };

static struct rtm_dump_nh_ctx *
rtm_dump_nh_ctx(struct netlink_callback *cb)
{ … }

static int rtm_dump_walk_nexthops(struct sk_buff *skb,
				  struct netlink_callback *cb,
				  struct rb_root *root,
				  struct rtm_dump_nh_ctx *ctx,
				  int (*nh_cb)(struct sk_buff *skb,
					       struct netlink_callback *cb,
					       struct nexthop *nh, void *data),
				  void *data)
{ … }

static int rtm_dump_nexthop_cb(struct sk_buff *skb, struct netlink_callback *cb,
			       struct nexthop *nh, void *data)
{ … }

/* rtnl */
static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
{ … }

static struct nexthop *
nexthop_find_group_resilient(struct net *net, u32 id,
			     struct netlink_ext_ack *extack)
{ … }

static int nh_valid_dump_nhid(struct nlattr *attr, u32 *nh_id_p,
			      struct netlink_ext_ack *extack)
{ … }

static int nh_valid_dump_bucket_req(const struct nlmsghdr *nlh,
				    struct nh_dump_filter *filter,
				    struct netlink_callback *cb)
{ … }

struct rtm_dump_res_bucket_ctx { … };

static struct rtm_dump_res_bucket_ctx *
rtm_dump_res_bucket_ctx(struct netlink_callback *cb)
{ … }

struct rtm_dump_nexthop_bucket_data { … };

static int rtm_dump_nexthop_bucket_nh(struct sk_buff *skb,
				      struct netlink_callback *cb,
				      struct nexthop *nh,
				      struct rtm_dump_nexthop_bucket_data *dd)
{ … }

static int rtm_dump_nexthop_bucket_cb(struct sk_buff *skb,
				      struct netlink_callback *cb,
				      struct nexthop *nh, void *data)
{ … }

/* rtnl */
static int rtm_dump_nexthop_bucket(struct sk_buff *skb,
				   struct netlink_callback *cb)
{ … }

static int nh_valid_get_bucket_req_res_bucket(struct nlattr *res,
					      u16 *bucket_index,
					      struct netlink_ext_ack *extack)
{ … }

static int nh_valid_get_bucket_req(const struct nlmsghdr *nlh,
				   u32 *id, u16 *bucket_index,
				   struct netlink_ext_ack *extack)
{ … }

/* rtnl */
static int rtm_get_nexthop_bucket(struct sk_buff *in_skb, struct nlmsghdr *nlh,
				  struct netlink_ext_ack *extack)
{ … }

static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
{ … }

/* rtnl */
static int nh_netdev_event(struct notifier_block *this,
			   unsigned long event, void *ptr)
{ … }

static struct notifier_block nh_netdev_notifier = …;

static int nexthops_dump(struct net *net, struct notifier_block *nb,
			 enum nexthop_event_type event_type,
			 struct netlink_ext_ack *extack)
{ … }

int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
			      struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);

int __unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
{ … }
EXPORT_SYMBOL(…);

int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
{ … }
EXPORT_SYMBOL(…);

void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
{ … }
EXPORT_SYMBOL(…);

void nexthop_bucket_set_hw_flags(struct net *net, u32 id, u16 bucket_index,
				 bool offload, bool trap)
{ … }
EXPORT_SYMBOL(…);

void nexthop_res_grp_activity_update(struct net *net, u32 id, u16 num_buckets,
				     unsigned long *activity)
{ … }
EXPORT_SYMBOL(…);

static void __net_exit nexthop_net_exit_batch_rtnl(struct list_head *net_list,
						   struct list_head *dev_to_kill)
{ … }

static void __net_exit nexthop_net_exit(struct net *net)
{ … }

static int __net_init nexthop_net_init(struct net *net)
{ … }

static struct pernet_operations nexthop_net_ops = …;

static int __init nexthop_init(void)
{ … }
subsys_initcall(nexthop_init);