#include <linux/err.h>
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
#include <linux/notifier.h>
#include <linux/netdevice.h>
#include <linux/netfilter.h>
#include <linux/module.h>
#include <linux/cache.h>
#include <linux/cpu.h>
#include <linux/audit.h>
#include <linux/rhashtable.h>
#include <linux/if_tunnel.h>
#include <linux/icmp.h>
#include <net/dst.h>
#include <net/flow.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>
#include <net/ip.h>
#include <net/gre.h>
#if IS_ENABLED(CONFIG_IPV6_MIP6)
#include <net/mip6.h>
#endif
#ifdef CONFIG_XFRM_STATISTICS
#include <net/snmp.h>
#endif
#ifdef CONFIG_XFRM_ESPINTCP
#include <net/espintcp.h>
#endif
#include "xfrm_hash.h"
#define XFRM_QUEUE_TMO_MIN …
#define XFRM_QUEUE_TMO_MAX …
#define XFRM_MAX_QUEUE_LEN …
struct xfrm_flo { … };
#define INEXACT_PREFIXLEN_IPV4 …
#define INEXACT_PREFIXLEN_IPV6 …
struct xfrm_pol_inexact_node { … };
struct xfrm_pol_inexact_key { … };
struct xfrm_pol_inexact_bin { … };
enum xfrm_pol_inexact_candidate_type { … };
struct xfrm_pol_inexact_candidates { … };
struct xfrm_flow_keys { … };
static struct flow_dissector xfrm_session_dissector __ro_after_init;
static DEFINE_SPINLOCK(xfrm_if_cb_lock);
static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
__read_mostly;
static struct kmem_cache *xfrm_dst_cache __ro_after_init;
static struct rhashtable xfrm_policy_inexact_table;
static const struct rhashtable_params xfrm_pol_inexact_params;
static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
static int stale_bundle(struct dst_entry *dst);
static int xfrm_bundle_ok(struct xfrm_dst *xdst);
static void xfrm_policy_queue_process(struct timer_list *t);
static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
int dir);
static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
u32 if_id);
static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup_rcu(struct net *net,
u8 type, u16 family, u8 dir, u32 if_id);
static struct xfrm_policy *
xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
bool excl);
static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
struct xfrm_policy *policy);
static bool
xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
struct xfrm_pol_inexact_bin *b,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr);
static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
{ … }
static inline bool
__xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
{ … }
static inline bool
__xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
{ … }
bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
unsigned short family)
{ … }
static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
{ … }
static const struct xfrm_if_cb *xfrm_if_get_cb(void)
{ … }
struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr,
int family, u32 mark)
{ … }
EXPORT_SYMBOL(…);
static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
int tos, int oif,
xfrm_address_t *prev_saddr,
xfrm_address_t *prev_daddr,
int family, u32 mark)
{ … }
static inline unsigned long make_jiffies(long secs)
{ … }
static void xfrm_policy_timer(struct timer_list *t)
{ … }
struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
{ … }
EXPORT_SYMBOL(…);
static void xfrm_policy_destroy_rcu(struct rcu_head *head)
{ … }
void xfrm_policy_destroy(struct xfrm_policy *policy)
{ … }
EXPORT_SYMBOL(…);
static void xfrm_policy_kill(struct xfrm_policy *policy)
{ … }
static unsigned int xfrm_policy_hashmax __read_mostly = …;
static inline unsigned int idx_hash(struct net *net, u32 index)
{ … }
static void __get_hash_thresh(struct net *net,
unsigned short family, int dir,
u8 *dbits, u8 *sbits)
{ … }
static struct hlist_head *policy_hash_bysel(struct net *net,
const struct xfrm_selector *sel,
unsigned short family, int dir)
{ … }
static struct hlist_head *policy_hash_direct(struct net *net,
const xfrm_address_t *daddr,
const xfrm_address_t *saddr,
unsigned short family, int dir)
{ … }
static void xfrm_dst_hash_transfer(struct net *net,
struct hlist_head *list,
struct hlist_head *ndsttable,
unsigned int nhashmask,
int dir)
{ … }
static void xfrm_idx_hash_transfer(struct hlist_head *list,
struct hlist_head *nidxtable,
unsigned int nhashmask)
{ … }
static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
{ … }
static void xfrm_bydst_resize(struct net *net, int dir)
{ … }
static void xfrm_byidx_resize(struct net *net)
{ … }
static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
{ … }
static inline int xfrm_byidx_should_resize(struct net *net, int total)
{ … }
void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
{ … }
EXPORT_SYMBOL(…);
static DEFINE_MUTEX(hash_resize_mutex);
static void xfrm_hash_resize(struct work_struct *work)
{ … }
static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
{ … }
static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
int family, u8 prefixlen)
{ … }
static bool
xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
{ … }
static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
const xfrm_address_t *addr, u8 prefixlen)
{ … }
static struct xfrm_pol_inexact_node *
xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
{ … }
static int xfrm_policy_addr_delta(const xfrm_address_t *a,
const xfrm_address_t *b,
u8 prefixlen, u16 family)
{ … }
static void xfrm_policy_inexact_list_reinsert(struct net *net,
struct xfrm_pol_inexact_node *n,
u16 family)
{ … }
static void xfrm_policy_inexact_node_reinsert(struct net *net,
struct xfrm_pol_inexact_node *n,
struct rb_root *new,
u16 family)
{ … }
static void xfrm_policy_inexact_node_merge(struct net *net,
struct xfrm_pol_inexact_node *v,
struct xfrm_pol_inexact_node *n,
u16 family)
{ … }
static struct xfrm_pol_inexact_node *
xfrm_policy_inexact_insert_node(struct net *net,
struct rb_root *root,
xfrm_address_t *addr,
u16 family, u8 prefixlen, u8 dir)
{ … }
static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
{ … }
static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
{ … }
static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
{ … }
static void __xfrm_policy_inexact_flush(struct net *net)
{ … }
static struct hlist_head *
xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
struct xfrm_policy *policy, u8 dir)
{ … }
static struct xfrm_policy *
xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
{ … }
static void xfrm_hash_rebuild(struct work_struct *work)
{ … }
void xfrm_policy_hash_rebuild(struct net *net)
{ … }
EXPORT_SYMBOL(…);
static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
{ … }
static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
{ … }
static void xfrm_policy_requeue(struct xfrm_policy *old,
struct xfrm_policy *new)
{ … }
static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark,
struct xfrm_policy *pol)
{ … }
static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
{ … }
static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
{ … }
static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
const void *ptr)
{ … }
static const struct rhashtable_params xfrm_pol_inexact_params = …;
static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
struct xfrm_policy *policy)
{ … }
static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
struct xfrm_policy *policy,
bool excl)
{ … }
int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
{ … }
EXPORT_SYMBOL(…);
static struct xfrm_policy *
__xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark,
u32 if_id, u8 type, int dir, struct xfrm_selector *sel,
struct xfrm_sec_ctx *ctx)
{ … }
struct xfrm_policy *
xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id,
u8 type, int dir, struct xfrm_selector *sel,
struct xfrm_sec_ctx *ctx, int delete, int *err)
{ … }
EXPORT_SYMBOL(…);
struct xfrm_policy *
xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id,
u8 type, int dir, u32 id, int delete, int *err)
{ … }
EXPORT_SYMBOL(…);
#ifdef CONFIG_SECURITY_NETWORK_XFRM
static inline int
xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
{ … }
static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
struct net_device *dev,
bool task_valid)
{ … }
#else
static inline int
xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
{
return 0;
}
static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
struct net_device *dev,
bool task_valid)
{
return 0;
}
#endif
int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
{ … }
EXPORT_SYMBOL(…);
int xfrm_dev_policy_flush(struct net *net, struct net_device *dev,
bool task_valid)
{ … }
EXPORT_SYMBOL(…);
int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
int (*func)(struct xfrm_policy *, int, int, void*),
void *data)
{ … }
EXPORT_SYMBOL(…);
void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
{ … }
EXPORT_SYMBOL(…);
void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
{ … }
EXPORT_SYMBOL(…);
static int xfrm_policy_match(const struct xfrm_policy *pol,
const struct flowi *fl,
u8 type, u16 family, u32 if_id)
{ … }
static struct xfrm_pol_inexact_node *
xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
seqcount_spinlock_t *count,
const xfrm_address_t *addr, u16 family)
{ … }
static bool
xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
struct xfrm_pol_inexact_bin *b,
const xfrm_address_t *saddr,
const xfrm_address_t *daddr)
{ … }
static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
u8 dir, u32 if_id)
{ … }
static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
u8 dir, u32 if_id)
{ … }
static struct xfrm_policy *
__xfrm_policy_eval_candidates(struct hlist_head *chain,
struct xfrm_policy *prefer,
const struct flowi *fl,
u8 type, u16 family, u32 if_id)
{ … }
static struct xfrm_policy *
xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
struct xfrm_policy *prefer,
const struct flowi *fl,
u8 type, u16 family, u32 if_id)
{ … }
static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
const struct flowi *fl,
u16 family, u8 dir,
u32 if_id)
{ … }
static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
const struct flowi *fl,
u16 family, u8 dir, u32 if_id)
{ … }
static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
const struct flowi *fl,
u16 family, u32 if_id)
{ … }
static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
{ … }
static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
int dir)
{ … }
static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
{ … }
static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
{ … }
int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
{ … }
EXPORT_SYMBOL(…);
int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
{ … }
static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
{ … }
int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
{ … }
static int
xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
xfrm_address_t *remote, unsigned short family, u32 mark)
{ … }
static int
xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
struct xfrm_state **xfrm, unsigned short family)
{ … }
static int
xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
struct xfrm_state **xfrm, unsigned short family)
{ … }
static int xfrm_get_tos(const struct flowi *fl, int family)
{ … }
static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
{ … }
static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
int nfheader_len)
{ … }
static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
const struct flowi *fl)
{ … }
static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
struct xfrm_state **xfrm,
struct xfrm_dst **bundle,
int nx,
const struct flowi *fl,
struct dst_entry *dst)
{ … }
static int xfrm_expand_policies(const struct flowi *fl, u16 family,
struct xfrm_policy **pols,
int *num_pols, int *num_xfrms)
{ … }
static struct xfrm_dst *
xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
const struct flowi *fl, u16 family,
struct dst_entry *dst_orig)
{ … }
static void xfrm_policy_queue_process(struct timer_list *t)
{ … }
static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{ … }
static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
struct xfrm_flo *xflo,
const struct flowi *fl,
int num_xfrms,
u16 family)
{ … }
static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
const struct flowi *fl,
u16 family, u8 dir,
struct xfrm_flo *xflo, u32 if_id)
{ … }
static struct dst_entry *make_blackhole(struct net *net, u16 family,
struct dst_entry *dst_orig)
{ … }
struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
struct dst_entry *dst_orig,
const struct flowi *fl,
const struct sock *sk,
int flags, u32 if_id)
{ … }
EXPORT_SYMBOL(…);
struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
const struct flowi *fl, const struct sock *sk,
int flags)
{ … }
EXPORT_SYMBOL(…);
struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
const struct flowi *fl,
const struct sock *sk, int flags)
{ … }
EXPORT_SYMBOL(…);
static inline int
xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
{ … }
static inline int
xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
unsigned short family, u32 if_id)
{ … }
static inline int
xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
unsigned short family, u32 if_id)
{ … }
static void
decode_session4(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
{ … }
#if IS_ENABLED(CONFIG_IPV6)
static void
decode_session6(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
{ … }
#endif
int __xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl,
unsigned int family, int reverse)
{ … }
EXPORT_SYMBOL(…);
static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
{ … }
static bool icmp_err_packet(const struct flowi *fl, unsigned short family)
{ … }
static bool xfrm_icmp_flow_decode(struct sk_buff *skb, unsigned short family,
const struct flowi *fl, struct flowi *fl1)
{ … }
static bool xfrm_selector_inner_icmp_match(struct sk_buff *skb, unsigned short family,
const struct xfrm_selector *sel,
const struct flowi *fl)
{ … }
static inline struct
xfrm_policy *xfrm_in_fwd_icmp(struct sk_buff *skb,
const struct flowi *fl, unsigned short family,
u32 if_id)
{ … }
static inline struct
dst_entry *xfrm_out_fwd_icmp(struct sk_buff *skb, struct flowi *fl,
unsigned short family, struct dst_entry *dst)
{ … }
int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
unsigned short family)
{ … }
EXPORT_SYMBOL(…);
int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
{ … }
EXPORT_SYMBOL(…);
static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
{ … }
static int stale_bundle(struct dst_entry *dst)
{ … }
void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
{ … }
EXPORT_SYMBOL(…);
static void xfrm_link_failure(struct sk_buff *skb)
{ … }
static void xfrm_negative_advice(struct sock *sk, struct dst_entry *dst)
{ … }
static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
{ … }
static int xfrm_bundle_ok(struct xfrm_dst *first)
{ … }
static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
{ … }
static unsigned int xfrm_mtu(const struct dst_entry *dst)
{ … }
static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
const void *daddr)
{ … }
static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
struct sk_buff *skb,
const void *daddr)
{ … }
static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
{ … }
int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
{ … }
EXPORT_SYMBOL(…);
void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
{ … }
EXPORT_SYMBOL(…);
void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
{ … }
EXPORT_SYMBOL(…);
void xfrm_if_unregister_cb(void)
{ … }
EXPORT_SYMBOL(…);
#ifdef CONFIG_XFRM_STATISTICS
static int __net_init xfrm_statistics_init(struct net *net)
{ … }
static void xfrm_statistics_fini(struct net *net)
{ … }
#else
static int __net_init xfrm_statistics_init(struct net *net)
{
return 0;
}
static void xfrm_statistics_fini(struct net *net)
{
}
#endif
static int __net_init xfrm_policy_init(struct net *net)
{ … }
static void xfrm_policy_fini(struct net *net)
{ … }
static int __net_init xfrm_net_init(struct net *net)
{ … }
static void __net_exit xfrm_net_exit(struct net *net)
{ … }
static struct pernet_operations __net_initdata xfrm_net_ops = …;
static const struct flow_dissector_key xfrm_flow_dissector_keys[] = …;
void __init xfrm_init(void)
{ … }
#ifdef CONFIG_AUDITSYSCALL
static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
struct audit_buffer *audit_buf)
{ … }
void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
{ … }
EXPORT_SYMBOL_GPL(…);
void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
bool task_valid)
{ … }
EXPORT_SYMBOL_GPL(…);
#endif
#ifdef CONFIG_XFRM_MIGRATE
static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
const struct xfrm_selector *sel_tgt)
{ … }
static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
u8 dir, u8 type, struct net *net, u32 if_id)
{ … }
static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
{ … }
static int xfrm_policy_migrate(struct xfrm_policy *pol,
struct xfrm_migrate *m, int num_migrate,
struct netlink_ext_ack *extack)
{ … }
static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate,
struct netlink_ext_ack *extack)
{ … }
int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
struct xfrm_migrate *m, int num_migrate,
struct xfrm_kmaddress *k, struct net *net,
struct xfrm_encap_tmpl *encap, u32 if_id,
struct netlink_ext_ack *extack)
{ … }
EXPORT_SYMBOL(…);
#endif