ebc0ffae5d
fib_lookup() converted to be called in RCU protected context, no reference taken and released on a contended cache line (fib_clntref) fib_table_lookup() and fib_semantic_match() get an additional parameter. struct fib_info gets an rcu_head field, and is freed after an rcu grace period. Stress test : (Sending 160.000.000 UDP frames on same neighbour, IP route cache disabled, dual E5540 @2.53GHz, 32bit kernel, FIB_HASH) (about same results for FIB_TRIE) Before patch : real 1m31.199s user 0m13.761s sys 23m24.780s After patch: real 1m5.375s user 0m14.997s sys 15m50.115s Before patch Profile : 13044.00 15.4% __ip_route_output_key vmlinux 8438.00 10.0% dst_destroy vmlinux 5983.00 7.1% fib_semantic_match vmlinux 5410.00 6.4% fib_rules_lookup vmlinux 4803.00 5.7% neigh_lookup vmlinux 4420.00 5.2% _raw_spin_lock vmlinux 3883.00 4.6% rt_set_nexthop vmlinux 3261.00 3.9% _raw_read_lock vmlinux 2794.00 3.3% fib_table_lookup vmlinux 2374.00 2.8% neigh_resolve_output vmlinux 2153.00 2.5% dst_alloc vmlinux 1502.00 1.8% _raw_read_lock_bh vmlinux 1484.00 1.8% kmem_cache_alloc vmlinux 1407.00 1.7% eth_header vmlinux 1406.00 1.7% ipv4_dst_destroy vmlinux 1298.00 1.5% __copy_from_user_ll vmlinux 1174.00 1.4% dev_queue_xmit vmlinux 1000.00 1.2% ip_output vmlinux After patch Profile : 13712.00 15.8% dst_destroy vmlinux 8548.00 9.9% __ip_route_output_key vmlinux 7017.00 8.1% neigh_lookup vmlinux 4554.00 5.3% fib_semantic_match vmlinux 4067.00 4.7% _raw_read_lock vmlinux 3491.00 4.0% dst_alloc vmlinux 3186.00 3.7% neigh_resolve_output vmlinux 3103.00 3.6% fib_table_lookup vmlinux 2098.00 2.4% _raw_read_lock_bh vmlinux 2081.00 2.4% kmem_cache_alloc vmlinux 2013.00 2.3% _raw_spin_lock vmlinux 1763.00 2.0% __copy_from_user_ll vmlinux 1763.00 2.0% ip_output vmlinux 1761.00 2.0% ipv4_dst_destroy vmlinux 1631.00 1.9% eth_header vmlinux 1440.00 1.7% _raw_read_unlock_bh vmlinux Reference results, if IP route cache is enabled : real 0m29.718s user 0m10.845s sys 7m37.341s 25213.00 29.5% __ip_route_output_key vmlinux 9011.00 10.5% dst_release vmlinux 4817.00 5.6% ip_push_pending_frames vmlinux 4232.00 5.0% ip_finish_output vmlinux 3940.00 4.6% udp_sendmsg vmlinux 3730.00 4.4% __copy_from_user_ll vmlinux 3716.00 4.4% ip_route_output_flow vmlinux 2451.00 2.9% __xfrm_lookup vmlinux 2221.00 2.6% ip_append_data vmlinux 1718.00 2.0% _raw_spin_lock_bh vmlinux 1655.00 1.9% __alloc_skb vmlinux 1572.00 1.8% sock_wfree vmlinux 1345.00 1.6% kfree vmlinux Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
120 lines
3 KiB
C
120 lines
3 KiB
C
#ifndef __NET_FIB_RULES_H
|
|
#define __NET_FIB_RULES_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/netdevice.h>
|
|
#include <linux/fib_rules.h>
|
|
#include <net/flow.h>
|
|
#include <net/rtnetlink.h>
|
|
|
|
struct fib_rule {
|
|
struct list_head list;
|
|
atomic_t refcnt;
|
|
int iifindex;
|
|
int oifindex;
|
|
u32 mark;
|
|
u32 mark_mask;
|
|
u32 pref;
|
|
u32 flags;
|
|
u32 table;
|
|
u8 action;
|
|
u32 target;
|
|
struct fib_rule * ctarget;
|
|
char iifname[IFNAMSIZ];
|
|
char oifname[IFNAMSIZ];
|
|
struct rcu_head rcu;
|
|
struct net * fr_net;
|
|
};
|
|
|
|
struct fib_lookup_arg {
|
|
void *lookup_ptr;
|
|
void *result;
|
|
struct fib_rule *rule;
|
|
int flags;
|
|
#define FIB_LOOKUP_NOREF 1
|
|
};
|
|
|
|
struct fib_rules_ops {
|
|
int family;
|
|
struct list_head list;
|
|
int rule_size;
|
|
int addr_size;
|
|
int unresolved_rules;
|
|
int nr_goto_rules;
|
|
|
|
int (*action)(struct fib_rule *,
|
|
struct flowi *, int,
|
|
struct fib_lookup_arg *);
|
|
int (*match)(struct fib_rule *,
|
|
struct flowi *, int);
|
|
int (*configure)(struct fib_rule *,
|
|
struct sk_buff *,
|
|
struct fib_rule_hdr *,
|
|
struct nlattr **);
|
|
int (*compare)(struct fib_rule *,
|
|
struct fib_rule_hdr *,
|
|
struct nlattr **);
|
|
int (*fill)(struct fib_rule *, struct sk_buff *,
|
|
struct fib_rule_hdr *);
|
|
u32 (*default_pref)(struct fib_rules_ops *ops);
|
|
size_t (*nlmsg_payload)(struct fib_rule *);
|
|
|
|
/* Called after modifications to the rules set, must flush
|
|
* the route cache if one exists. */
|
|
void (*flush_cache)(struct fib_rules_ops *ops);
|
|
|
|
int nlgroup;
|
|
const struct nla_policy *policy;
|
|
struct list_head rules_list;
|
|
struct module *owner;
|
|
struct net *fro_net;
|
|
struct rcu_head rcu;
|
|
};
|
|
|
|
#define FRA_GENERIC_POLICY \
|
|
[FRA_IIFNAME] = { .type = NLA_STRING, .len = IFNAMSIZ - 1 }, \
|
|
[FRA_OIFNAME] = { .type = NLA_STRING, .len = IFNAMSIZ - 1 }, \
|
|
[FRA_PRIORITY] = { .type = NLA_U32 }, \
|
|
[FRA_FWMARK] = { .type = NLA_U32 }, \
|
|
[FRA_FWMASK] = { .type = NLA_U32 }, \
|
|
[FRA_TABLE] = { .type = NLA_U32 }, \
|
|
[FRA_GOTO] = { .type = NLA_U32 }
|
|
|
|
static inline void fib_rule_get(struct fib_rule *rule)
|
|
{
|
|
atomic_inc(&rule->refcnt);
|
|
}
|
|
|
|
static inline void fib_rule_put_rcu(struct rcu_head *head)
|
|
{
|
|
struct fib_rule *rule = container_of(head, struct fib_rule, rcu);
|
|
release_net(rule->fr_net);
|
|
kfree(rule);
|
|
}
|
|
|
|
static inline void fib_rule_put(struct fib_rule *rule)
|
|
{
|
|
if (atomic_dec_and_test(&rule->refcnt))
|
|
call_rcu(&rule->rcu, fib_rule_put_rcu);
|
|
}
|
|
|
|
static inline u32 frh_get_table(struct fib_rule_hdr *frh, struct nlattr **nla)
|
|
{
|
|
if (nla[FRA_TABLE])
|
|
return nla_get_u32(nla[FRA_TABLE]);
|
|
return frh->table;
|
|
}
|
|
|
|
extern struct fib_rules_ops *fib_rules_register(const struct fib_rules_ops *, struct net *);
|
|
extern void fib_rules_unregister(struct fib_rules_ops *);
|
|
|
|
extern int fib_rules_lookup(struct fib_rules_ops *,
|
|
struct flowi *, int flags,
|
|
struct fib_lookup_arg *);
|
|
extern int fib_default_rule_add(struct fib_rules_ops *,
|
|
u32 pref, u32 table,
|
|
u32 flags);
|
|
extern u32 fib_default_rule_pref(struct fib_rules_ops *ops);
|
|
#endif
|