kernel_optimize_test/include/net/ip6_fib.h

289 lines
6.4 KiB
C
Raw Normal View History

/*
* Linux INET6 implementation
*
* Authors:
* Pedro Roque <roque@di.fc.ul.pt>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#ifndef _IP6_FIB_H
#define _IP6_FIB_H
#include <linux/ipv6_route.h>
#include <linux/rtnetlink.h>
#include <linux/spinlock.h>
#include <net/dst.h>
#include <net/flow.h>
#include <net/netlink.h>
#include <net/inetpeer.h>
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
#define FIB6_TABLE_HASHSZ 256
#else
#define FIB6_TABLE_HASHSZ 1
#endif
struct rt6_info;
struct fib6_config {
u32 fc_table;
u32 fc_metric;
int fc_dst_len;
int fc_src_len;
int fc_ifindex;
u32 fc_flags;
u32 fc_protocol;
struct in6_addr fc_dst;
struct in6_addr fc_src;
struct in6_addr fc_prefsrc;
struct in6_addr fc_gateway;
unsigned long fc_expires;
struct nlattr *fc_mx;
int fc_mx_len;
struct nl_info fc_nlinfo;
};
struct fib6_node {
struct fib6_node *parent;
struct fib6_node *left;
struct fib6_node *right;
#ifdef CONFIG_IPV6_SUBTREES
struct fib6_node *subtree;
#endif
struct rt6_info *leaf;
__u16 fn_bit; /* bit key */
__u16 fn_flags;
__u32 fn_sernum;
[IPV6]: Fix routing round-robin locking. As per RFC2461, section 6.3.6, item #2, when no routers on the matching list are known to be reachable or probably reachable we do round robin on those available routes so that we make sure to probe as many of them as possible to detect when one becomes reachable faster. Each routing table has a rwlock protecting the tree and the linked list of routes at each leaf. The round robin code executes during lookup and thus with the rwlock taken as a reader. A small local spinlock tries to provide protection but this does not work at all for two reasons: 1) The round-robin list manipulation, as coded, goes like this (with read lock held): walk routes finding head and tail spin_lock(); rotate list using head and tail spin_unlock(); While one thread is rotating the list, another thread can end up with stale values of head and tail and then proceed to corrupt the list when it gets the lock. This ends up causing the OOPS in fib6_add() later onthat many people have been hitting. 2) All the other code paths that run with the rwlock held as a reader do not expect the list to change on them, they expect it to remain completely fixed while they hold the lock in that way. So, simply stated, it is impossible to implement this correctly using a manipulation of the list without violating the rwlock locking semantics. Reimplement using a per-fib6_node round-robin pointer. This way we don't need to manipulate the list at all, and since the round-robin pointer can only ever point to real existing entries we don't need to perform any locking on the changing of the round-robin pointer itself. We only need to reset the round-robin pointer to NULL when the entry it is pointing to is removed. The idea is from Thomas Graf and it is very similar to how this was implemented before the advanced router selection code when in. Signed-off-by: David S. Miller <davem@davemloft.net>
2007-03-25 11:36:25 +08:00
struct rt6_info *rr_ptr;
};
#ifndef CONFIG_IPV6_SUBTREES
#define FIB6_SUBTREE(fn) NULL
#else
#define FIB6_SUBTREE(fn) ((fn)->subtree)
#endif
/*
* routing information
*
*/
struct rt6key {
struct in6_addr addr;
int plen;
};
struct fib6_table;
struct rt6_info {
struct dst_entry dst;
/*
* Tail elements of dst_entry (__refcnt etc.)
* and these elements (rarely used in hot path) are in
* the same cache line.
*/
struct fib6_table *rt6i_table;
struct fib6_node *rt6i_node;
struct in6_addr rt6i_gateway;
atomic_t rt6i_ref;
/* These are in a separate cache line. */
struct rt6key rt6i_dst ____cacheline_aligned_in_smp;
u32 rt6i_flags;
struct rt6key rt6i_src;
struct rt6key rt6i_prefsrc;
u32 rt6i_metric;
inet: Create a mechanism for upward inetpeer propagation into routes. If we didn't have a routing cache, we would not be able to properly propagate certain kinds of dynamic path attributes, for example PMTU information and redirects. The reason is that if we didn't have a routing cache, then there would be no way to lookup all of the active cached routes hanging off of sockets, tunnels, IPSEC bundles, etc. Consider the case where we created a cached route, but no inetpeer entry existed and also we were not asked to pre-COW the route metrics and therefore did not force the creation a new inetpeer entry. If we later get a PMTU message, or a redirect, and store this information in a new inetpeer entry, there is no way to teach that cached route about the newly existing inetpeer entry. The facilities implemented here handle this problem. First we create a generation ID. When we create a cached route of any kind, we remember the generation ID at the time of attachment. Any time we force-create an inetpeer entry in response to new path information, we bump that generation ID. The dst_ops->check() callback is where the knowledge of this event is propagated. If the global generation ID does not equal the one stored in the cached route, and the cached route has not attached to an inetpeer yet, we look it up and attach if one is found. Now that we've updated the cached route's information, we update the route's generation ID too. This clears the way for implementing PMTU and redirects directly in the inetpeer cache. There is absolutely no need to consult cached route information in order to maintain this information. At this point nothing bumps the inetpeer genids, that comes in the later changes which handle PMTUs and redirects using inetpeers. Signed-off-by: David S. Miller <davem@davemloft.net>
2011-02-08 12:38:06 +08:00
u32 rt6i_peer_genid;
struct inet6_dev *rt6i_idev;
struct inet_peer *rt6i_peer;
#ifdef CONFIG_XFRM
u32 rt6i_flow_cache_genid;
#endif
/* more non-fragment space at head required */
unsigned short rt6i_nfheader_len;
u8 rt6i_protocol;
};
static inline struct inet6_dev *ip6_dst_idev(struct dst_entry *dst)
{
return ((struct rt6_info *)dst)->rt6i_idev;
}
static inline void rt6_clean_expires(struct rt6_info *rt)
{
if (!(rt->rt6i_flags & RTF_EXPIRES) && rt->dst.from)
dst_release(rt->dst.from);
rt->rt6i_flags &= ~RTF_EXPIRES;
rt->dst.from = NULL;
}
static inline void rt6_set_expires(struct rt6_info *rt, unsigned long expires)
{
if (!(rt->rt6i_flags & RTF_EXPIRES) && rt->dst.from)
dst_release(rt->dst.from);
rt->rt6i_flags |= RTF_EXPIRES;
rt->dst.expires = expires;
}
static inline void rt6_update_expires(struct rt6_info *rt, int timeout)
{
if (!(rt->rt6i_flags & RTF_EXPIRES)) {
if (rt->dst.from)
dst_release(rt->dst.from);
/* dst_set_expires relies on expires == 0
* if it has not been set previously.
*/
rt->dst.expires = 0;
}
dst_set_expires(&rt->dst, timeout);
rt->rt6i_flags |= RTF_EXPIRES;
}
static inline void rt6_set_from(struct rt6_info *rt, struct rt6_info *from)
{
struct dst_entry *new = (struct dst_entry *) from;
if (!(rt->rt6i_flags & RTF_EXPIRES) && rt->dst.from) {
if (new == rt->dst.from)
return;
dst_release(rt->dst.from);
}
rt->rt6i_flags &= ~RTF_EXPIRES;
rt->dst.from = new;
dst_hold(new);
}
struct fib6_walker_t {
struct list_head lh;
struct fib6_node *root, *node;
struct rt6_info *leaf;
unsigned char state;
unsigned char prune;
ipv6: fib: fix crash when changing large fib while dumping it When the fib size exceeds what can be dumped in a single skb, the dump is suspended and resumed once the last skb has been received by userspace. When the fib is changed while the dump is suspended, the walker might contain stale pointers, causing a crash when the dump is resumed. BUG: unable to handle kernel NULL pointer dereference at 0000000000000018 IP: [<ffffffffa01bce04>] fib6_walk_continue+0xbb/0x124 [ipv6] PGD 5347a067 PUD 65c7067 PMD 0 Oops: 0000 [#1] PREEMPT SMP ... RIP: 0010:[<ffffffffa01bce04>] [<ffffffffa01bce04>] fib6_walk_continue+0xbb/0x124 [ipv6] ... Call Trace: [<ffffffff8104aca3>] ? mutex_spin_on_owner+0x59/0x71 [<ffffffffa01bd105>] inet6_dump_fib+0x11b/0x1b9 [ipv6] [<ffffffff81371af4>] netlink_dump+0x5b/0x19e [<ffffffff8134f288>] ? consume_skb+0x28/0x2a [<ffffffff81373b69>] netlink_recvmsg+0x1ab/0x2c6 [<ffffffff81372781>] ? netlink_unicast+0xfa/0x151 [<ffffffff813483e0>] __sock_recvmsg+0x6d/0x79 [<ffffffff81348a53>] sock_recvmsg+0xca/0xe3 [<ffffffff81066d4b>] ? autoremove_wake_function+0x0/0x38 [<ffffffff811ed1f8>] ? radix_tree_lookup_slot+0xe/0x10 [<ffffffff810b3ed7>] ? find_get_page+0x90/0xa5 [<ffffffff810b5dc5>] ? filemap_fault+0x201/0x34f [<ffffffff810ef152>] ? fget_light+0x2f/0xac [<ffffffff813519e7>] ? verify_iovec+0x4f/0x94 [<ffffffff81349a65>] sys_recvmsg+0x14d/0x223 Store the serial number when beginning to walk the fib and reload pointers when continuing to walk after a change occured. Similar to other dumping functions, this might cause unrelated entries to be missed when entries are deleted. Tested-by: Ben Greear <greearb@candelatech.com> Signed-off-by: Patrick McHardy <kaber@trash.net> Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-08 13:19:03 +08:00
unsigned int skip;
unsigned int count;
int (*func)(struct fib6_walker_t *);
void *args;
};
struct rt6_statistics {
__u32 fib_nodes;
__u32 fib_route_nodes;
__u32 fib_rt_alloc; /* permanent routes */
__u32 fib_rt_entries; /* rt entries in table */
__u32 fib_rt_cache; /* cache routes */
__u32 fib_discarded_routes;
};
#define RTN_TL_ROOT 0x0001
#define RTN_ROOT 0x0002 /* tree root node */
#define RTN_RTINFO 0x0004 /* node with valid routing info */
/*
* priority levels (or metrics)
*
*/
struct fib6_table {
struct hlist_node tb6_hlist;
u32 tb6_id;
rwlock_t tb6_lock;
struct fib6_node tb6_root;
};
#define RT6_TABLE_UNSPEC RT_TABLE_UNSPEC
#define RT6_TABLE_MAIN RT_TABLE_MAIN
#define RT6_TABLE_DFLT RT6_TABLE_MAIN
#define RT6_TABLE_INFO RT6_TABLE_MAIN
#define RT6_TABLE_PREFIX RT6_TABLE_MAIN
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
#define FIB6_TABLE_MIN 1
#define FIB6_TABLE_MAX RT_TABLE_MAX
#define RT6_TABLE_LOCAL RT_TABLE_LOCAL
#else
#define FIB6_TABLE_MIN RT_TABLE_MAIN
#define FIB6_TABLE_MAX FIB6_TABLE_MIN
#define RT6_TABLE_LOCAL RT6_TABLE_MAIN
#endif
typedef struct rt6_info *(*pol_lookup_t)(struct net *,
struct fib6_table *,
struct flowi6 *, int);
/*
* exported functions
*/
extern struct fib6_table *fib6_get_table(struct net *net, u32 id);
extern struct fib6_table *fib6_new_table(struct net *net, u32 id);
extern struct dst_entry *fib6_rule_lookup(struct net *net,
struct flowi6 *fl6, int flags,
pol_lookup_t lookup);
extern struct fib6_node *fib6_lookup(struct fib6_node *root,
const struct in6_addr *daddr,
const struct in6_addr *saddr);
struct fib6_node *fib6_locate(struct fib6_node *root,
const struct in6_addr *daddr, int dst_len,
const struct in6_addr *saddr, int src_len);
IPv6: Avoid taking write lock for /proc/net/ipv6_route During some debugging I needed to look into how /proc/net/ipv6_route operated and in my digging I found its calling fib6_clean_all() which uses "write_lock_bh(&table->tb6_lock)" before doing the walk of the table. I found this on 2.6.32, but reading the code I believe the same basic idea exists currently. Looking at the rtnetlink code they are only calling "read_lock_bh(&table->tb6_lock);" via fib6_dump_table(). While I realize reading from proc isn't the recommended way of fetching the ipv6 route table; taking a write lock seems unnecessary and would probably cause network performance issues. To verify this I loaded up the ipv6 route table and then ran iperf in 3 cases: * doing nothing * reading ipv6 route table via proc (while :; do cat /proc/net/ipv6_route > /dev/null; done) * reading ipv6 route table via rtnetlink (while :; do ip -6 route show table all > /dev/null; done) * Load the ipv6 route table up with: * for ((i = 0;i < 4000;i++)); do ip route add unreachable 2000::$i; done * iperf commands: * client: iperf -i 1 -V -c <ipv6 addr> * server: iperf -V -s * iperf results - 3 runs each (in Mbits/sec) * nothing: client: 927,927,927 server: 927,927,927 * proc: client: 179,97,96,113 server: 142,112,133 * iproute: client: 928,927,928 server: 927,927,927 lock_stat shows taking the write lock is causing the slowdown. Using this info I decided to write a version of fib6_clean_all() which replaces write_lock_bh(&table->tb6_lock) with read_lock_bh(&table->tb6_lock). With this new function I see the same results as with my rtnetlink iperf test. Signed-off-by: Josh Hunt <joshhunt00@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
2011-12-28 21:23:07 +08:00
extern void fib6_clean_all_ro(struct net *net,
int (*func)(struct rt6_info *, void *arg),
int prune, void *arg);
extern void fib6_clean_all(struct net *net,
int (*func)(struct rt6_info *, void *arg),
int prune, void *arg);
extern int fib6_add(struct fib6_node *root,
struct rt6_info *rt,
struct nl_info *info);
extern int fib6_del(struct rt6_info *rt,
struct nl_info *info);
extern void inet6_rt_notify(int event, struct rt6_info *rt,
struct nl_info *info);
extern void fib6_run_gc(unsigned long expires,
struct net *net);
extern void fib6_gc_cleanup(void);
extern int fib6_init(void);
#ifdef CONFIG_IPV6_MULTIPLE_TABLES
extern int fib6_rules_init(void);
extern void fib6_rules_cleanup(void);
#else
static inline int fib6_rules_init(void)
{
return 0;
}
static inline void fib6_rules_cleanup(void)
{
return ;
}
#endif
#endif