Return to fib_hash.c CVS log

Up to [Development] / linux-2.6-xfs / net / ipv4

Initial Import 2.6.0

/*
 * INET		An implementation of the TCP/IP protocol suite for the LINUX
 *		operating system.  INET is implemented using the  BSD Socket
 *		interface as the means of communication with the user level.
 *
 *		IPv4 FIB: lookup engine and maintenance routines.
 *
 * Version:	$Id: fib_hash.c,v 1.13 2001/10/31 21:55:54 davem Exp $
 *
 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 */

#include <linux/config.h>
#include <asm/uaccess.h>
#include <asm/system.h>
#include <asm/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/proc_fs.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/init.h>

#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
#include <net/tcp.h>
#include <net/sock.h>
#include <net/ip_fib.h>

#define FTprint(a...)
/*
   printk(KERN_DEBUG a)
 */

static kmem_cache_t * fn_hash_kmem;

/*
   These bizarre types are just to force strict type checking.
   When I reversed order of bytes and changed to natural mask lengths,
   I forgot to make fixes in several places. Now I am lazy to return
   it back.
 */

typedef struct {
	u32	datum;
} fn_key_t;

typedef struct {
	u32	datum;
} fn_hash_idx_t;

struct fib_node
{
	struct fib_node		*fn_next;
	struct fib_info		*fn_info;
#define FIB_INFO(f)	((f)->fn_info)
	fn_key_t		fn_key;
	u8			fn_tos;
	u8			fn_type;
	u8			fn_scope;
	u8			fn_state;
};

#define FN_S_ZOMBIE	1
#define FN_S_ACCESSED	2

static int fib_hash_zombies;

struct fn_zone
{
	struct fn_zone	*fz_next;	/* Next not empty zone	*/
	struct fib_node	**fz_hash;	/* Hash table pointer	*/
	int		fz_nent;	/* Number of entries	*/

	int		fz_divisor;	/* Hash divisor		*/
	u32		fz_hashmask;	/* (fz_divisor - 1)	*/
#define FZ_HASHMASK(fz)	((fz)->fz_hashmask)

	int		fz_order;	/* Zone order		*/
	u32		fz_mask;
#define FZ_MASK(fz)	((fz)->fz_mask)
};

/* NOTE. On fast computers evaluation of fz_hashmask and fz_mask
   can be cheaper than memory lookup, so that FZ_* macros are used.
 */

struct fn_hash
{
	struct fn_zone	*fn_zones[33];
	struct fn_zone	*fn_zone_list;
};

static __inline__ fn_hash_idx_t fn_hash(fn_key_t key, struct fn_zone *fz)
{
	u32 h = ntohl(key.datum)>>(32 - fz->fz_order);
	h ^= (h>>20);
	h ^= (h>>10);
	h ^= (h>>5);
	h &= FZ_HASHMASK(fz);
	return *(fn_hash_idx_t*)&h;
}

#define fz_key_0(key)		((key).datum = 0)
#define fz_prefix(key,fz)	((key).datum)

static __inline__ fn_key_t fz_key(u32 dst, struct fn_zone *fz)
{
	fn_key_t k;
	k.datum = dst & FZ_MASK(fz);
	return k;
}

static __inline__ struct fib_node ** fz_chain_p(fn_key_t key, struct fn_zone *fz)
{
	return &fz->fz_hash[fn_hash(key, fz).datum];
}

static __inline__ struct fib_node * fz_chain(fn_key_t key, struct fn_zone *fz)
{
	return fz->fz_hash[fn_hash(key, fz).datum];
}

static __inline__ int fn_key_eq(fn_key_t a, fn_key_t b)
{
	return a.datum == b.datum;
}

static __inline__ int fn_key_leq(fn_key_t a, fn_key_t b)
{
	return a.datum <= b.datum;
}

static rwlock_t fib_hash_lock = RW_LOCK_UNLOCKED;

#define FZ_MAX_DIVISOR ((PAGE_SIZE<<MAX_ORDER) / sizeof(struct fib_node *))

static struct fib_node **fz_hash_alloc(int divisor)
{
	unsigned long size = divisor * sizeof(struct fib_node *);

	if (divisor <= 1024) {
		return kmalloc(size, GFP_KERNEL);
	} else {
		return (struct fib_node **)
			__get_free_pages(GFP_KERNEL, get_order(size));
	}
}

/* The fib hash lock must be held when this is called. */
static __inline__ void fn_rebuild_zone(struct fn_zone *fz,
				       struct fib_node **old_ht,
				       int old_divisor)
{
	int i;
	struct fib_node *f, **fp, *next;

	for (i=0; i<old_divisor; i++) {
		for (f=old_ht[i]; f; f=next) {
			next = f->fn_next;
			for (fp = fz_chain_p(f->fn_key, fz);
			     *fp && fn_key_leq((*fp)->fn_key, f->fn_key);
			     fp = &(*fp)->fn_next)
				/* NONE */;
			f->fn_next = *fp;
			*fp = f;
		}
	}
}

static void fz_hash_free(struct fib_node **hash, int divisor)
{
	if (divisor <= 1024)
		kfree(hash);
	else
		free_pages((unsigned long) hash,
			   get_order(divisor * sizeof(struct fib_node *)));
}

static void fn_rehash_zone(struct fn_zone *fz)
{
	struct fib_node **ht, **old_ht;
	int old_divisor, new_divisor;
	u32 new_hashmask;
		
	old_divisor = fz->fz_divisor;

	switch (old_divisor) {
	case 16:
		new_divisor = 256;
		break;
	case 256:
		new_divisor = 1024;
		break;
	default:
		if ((old_divisor << 1) > FZ_MAX_DIVISOR) {
			printk(KERN_CRIT "route.c: bad divisor %d!\n", old_divisor);
			return;
		}
		new_divisor = (old_divisor << 1);
		break;
	}

	new_hashmask = (new_divisor - 1);

#if RT_CACHE_DEBUG >= 2
	printk("fn_rehash_zone: hash for zone %d grows from %d\n", fz->fz_order, old_divisor);
#endif

	ht = fz_hash_alloc(new_divisor);

	if (ht)	{
		memset(ht, 0, new_divisor*sizeof(struct fib_node*));

		write_lock_bh(&fib_hash_lock);
		old_ht = fz->fz_hash;
		fz->fz_hash = ht;
		fz->fz_hashmask = new_hashmask;
		fz->fz_divisor = new_divisor;
		fn_rebuild_zone(fz, old_ht, old_divisor);
		write_unlock_bh(&fib_hash_lock);

		fz_hash_free(old_ht, old_divisor);
	}
}

static void fn_free_node(struct fib_node * f)
{
	fib_release_info(FIB_INFO(f));
	kmem_cache_free(fn_hash_kmem, f);
}


static struct fn_zone *
fn_new_zone(struct fn_hash *table, int z)
{
	int i;
	struct fn_zone *fz = kmalloc(sizeof(struct fn_zone), GFP_KERNEL);
	if (!fz)
		return NULL;

	memset(fz, 0, sizeof(struct fn_zone));
	if (z) {
		fz->fz_divisor = 16;
	} else {
		fz->fz_divisor = 1;
	}
	fz->fz_hashmask = (fz->fz_divisor - 1);
	fz->fz_hash = fz_hash_alloc(fz->fz_divisor);
	if (!fz->fz_hash) {
		kfree(fz);
		return NULL;
	}
	memset(fz->fz_hash, 0, fz->fz_divisor*sizeof(struct fib_node*));
	fz->fz_order = z;
	fz->fz_mask = inet_make_mask(z);

	/* Find the first not empty zone with more specific mask */
	for (i=z+1; i<=32; i++)
		if (table->fn_zones[i])
			break;
	write_lock_bh(&fib_hash_lock);
	if (i>32) {
		/* No more specific masks, we are the first. */
		fz->fz_next = table->fn_zone_list;
		table->fn_zone_list = fz;
	} else {
		fz->fz_next = table->fn_zones[i]->fz_next;
		table->fn_zones[i]->fz_next = fz;
	}
	table->fn_zones[z] = fz;
	write_unlock_bh(&fib_hash_lock);
	return fz;
}

static int
fn_hash_lookup(struct fib_table *tb, const struct flowi *flp, struct fib_result *res)
{
	int err;
	struct fn_zone *fz;
	struct fn_hash *t = (struct fn_hash*)tb->tb_data;

	read_lock(&fib_hash_lock);
	for (fz = t->fn_zone_list; fz; fz = fz->fz_next) {
		struct fib_node *f;
		fn_key_t k = fz_key(flp->fl4_dst, fz);

		for (f = fz_chain(k, fz); f; f = f->fn_next) {
			if (!fn_key_eq(k, f->fn_key)) {
				if (fn_key_leq(k, f->fn_key))
					break;
				else
					continue;
			}
#ifdef CONFIG_IP_ROUTE_TOS
			if (f->fn_tos && f->fn_tos != flp->fl4_tos)
				continue;
#endif
			f->fn_state |= FN_S_ACCESSED;

			if (f->fn_state&FN_S_ZOMBIE)
				continue;
			if (f->fn_scope < flp->fl4_scope)
				continue;

			err = fib_semantic_match(f->fn_type, FIB_INFO(f), flp, res);
			if (err == 0) {
				res->type = f->fn_type;
				res->scope = f->fn_scope;
				res->prefixlen = fz->fz_order;
				goto out;
			}
			if (err < 0)
				goto out;
		}
	}
	err = 1;
out:
	read_unlock(&fib_hash_lock);
	return err;
}

static int fn_hash_last_dflt=-1;

static int fib_detect_death(struct fib_info *fi, int order,
			    struct fib_info **last_resort, int *last_idx)
{
	struct neighbour *n;
	int state = NUD_NONE;

	n = neigh_lookup(&arp_tbl, &fi->fib_nh[0].nh_gw, fi->fib_dev);
	if (n) {
		state = n->nud_state;
		neigh_release(n);
	}
	if (state==NUD_REACHABLE)
		return 0;
	if ((state&NUD_VALID) && order != fn_hash_last_dflt)
		return 0;
	if ((state&NUD_VALID) ||
	    (*last_idx<0 && order > fn_hash_last_dflt)) {
		*last_resort = fi;
		*last_idx = order;
	}
	return 1;
}

static void
fn_hash_select_default(struct fib_table *tb, const struct flowi *flp, struct fib_result *res)
{
	int order, last_idx;
	struct fib_node *f;
	struct fib_info *fi = NULL;
	struct fib_info *last_resort;
	struct fn_hash *t = (struct fn_hash*)tb->tb_data;
	struct fn_zone *fz = t->fn_zones[0];

	if (fz == NULL)
		return;

	last_idx = -1;
	last_resort = NULL;
	order = -1;

	read_lock(&fib_hash_lock);
	for (f = fz->fz_hash[0]; f; f = f->fn_next) {
		struct fib_info *next_fi = FIB_INFO(f);

		if ((f->fn_state&FN_S_ZOMBIE) ||
		    f->fn_scope != res->scope ||
		    f->fn_type != RTN_UNICAST)
			continue;

		if (next_fi->fib_priority > res->fi->fib_priority)
			break;
		if (!next_fi->fib_nh[0].nh_gw || next_fi->fib_nh[0].nh_scope != RT_SCOPE_LINK)
			continue;
		f->fn_state |= FN_S_ACCESSED;

		if (fi == NULL) {
			if (next_fi != res->fi)
				break;
		} else if (!fib_detect_death(fi, order, &last_resort, &last_idx)) {
			if (res->fi)
				fib_info_put(res->fi);
			res->fi = fi;
			atomic_inc(&fi->fib_clntref);
			fn_hash_last_dflt = order;
			goto out;
		}
		fi = next_fi;
		order++;
	}

	if (order<=0 || fi==NULL) {
		fn_hash_last_dflt = -1;
		goto out;
	}

	if (!fib_detect_death(fi, order, &last_resort, &last_idx)) {
		if (res->fi)
			fib_info_put(res->fi);
		res->fi = fi;
		atomic_inc(&fi->fib_clntref);
		fn_hash_last_dflt = order;
		goto out;
	}

	if (last_idx >= 0) {
		if (res->fi)
			fib_info_put(res->fi);
		res->fi = last_resort;
		if (last_resort)
			atomic_inc(&last_resort->fib_clntref);
	}
	fn_hash_last_dflt = last_idx;
out:
	read_unlock(&fib_hash_lock);
}

#define FIB_SCAN(f, fp) \
for ( ; ((f) = *(fp)) != NULL; (fp) = &(f)->fn_next)

#define FIB_SCAN_KEY(f, fp, key) \
for ( ; ((f) = *(fp)) != NULL && fn_key_eq((f)->fn_key, (key)); (fp) = &(f)->fn_next)

#ifndef CONFIG_IP_ROUTE_TOS
#define FIB_SCAN_TOS(f, fp, key, tos) FIB_SCAN_KEY(f, fp, key)
#else
#define FIB_SCAN_TOS(f, fp, key, tos) \
for ( ; ((f) = *(fp)) != NULL && fn_key_eq((f)->fn_key, (key)) && \
     (f)->fn_tos == (tos) ; (fp) = &(f)->fn_next)
#endif


static void rtmsg_fib(int, struct fib_node*, int, int,
		      struct nlmsghdr *n,
		      struct netlink_skb_parms *);

static int
fn_hash_insert(struct fib_table *tb, struct rtmsg *r, struct kern_rta *rta,
		struct nlmsghdr *n, struct netlink_skb_parms *req)
{
	struct fn_hash *table = (struct fn_hash*)tb->tb_data;
	struct fib_node *new_f, *f, **fp, **del_fp;
	struct fn_zone *fz;
	struct fib_info *fi;

	int z = r->rtm_dst_len;
	int type = r->rtm_type;
#ifdef CONFIG_IP_ROUTE_TOS
	u8 tos = r->rtm_tos;
#endif
	fn_key_t key;
	int err;

FTprint("tb(%d)_insert: %d %08x/%d %d %08x\n", tb->tb_id, r->rtm_type, rta->rta_dst ?
*(u32*)rta->rta_dst : 0, z, rta->rta_oif ? *rta->rta_oif : -1,
rta->rta_prefsrc ? *(u32*)rta->rta_prefsrc : 0);
	if (z > 32)
		return -EINVAL;
	fz = table->fn_zones[z];
	if (!fz && !(fz = fn_new_zone(table, z)))
		return -ENOBUFS;

	fz_key_0(key);
	if (rta->rta_dst) {
		u32 dst;
		memcpy(&dst, rta->rta_dst, 4);
		if (dst & ~FZ_MASK(fz))
			return -EINVAL;
		key = fz_key(dst, fz);
	}

	if  ((fi = fib_create_info(r, rta, n, &err)) == NULL)
		return err;

	if (fz->fz_nent > (fz->fz_divisor<<1) &&
	    fz->fz_divisor < FZ_MAX_DIVISOR &&
	    (z==32 || (1<<z) > fz->fz_divisor))
		fn_rehash_zone(fz);

	fp = fz_chain_p(key, fz);


	/*
	 * Scan list to find the first route with the same destination
	 */
	FIB_SCAN(f, fp) {
		if (fn_key_leq(key,f->fn_key))
			break;
	}

#ifdef CONFIG_IP_ROUTE_TOS
	/*
	 * Find route with the same destination and tos.
	 */
	FIB_SCAN_KEY(f, fp, key) {
		if (f->fn_tos <= tos)
			break;
	}
#endif

	del_fp = NULL;

	if (f && (f->fn_state&FN_S_ZOMBIE) &&
#ifdef CONFIG_IP_ROUTE_TOS
	    f->fn_tos == tos &&
#endif
	    fn_key_eq(f->fn_key, key)) {
		del_fp = fp;
		fp = &f->fn_next;
		f = *fp;
		goto create;
	}

	FIB_SCAN_TOS(f, fp, key, tos) {
		if (fi->fib_priority <= FIB_INFO(f)->fib_priority)
			break;
	}

	/* Now f==*fp points to the first node with the same
	   keys [prefix,tos,priority], if such key already
	   exists or to the node, before which we will insert new one.
	 */

	if (f && 
#ifdef CONFIG_IP_ROUTE_TOS
	    f->fn_tos == tos &&
#endif
	    fn_key_eq(f->fn_key, key) &&
	    fi->fib_priority == FIB_INFO(f)->fib_priority) {
		struct fib_node **ins_fp;

		err = -EEXIST;
		if (n->nlmsg_flags&NLM_F_EXCL)
			goto out;

		if (n->nlmsg_flags&NLM_F_REPLACE) {
			del_fp = fp;
			fp = &f->fn_next;
			f = *fp;
			goto replace;
		}

		ins_fp = fp;
		err = -EEXIST;

		FIB_SCAN_TOS(f, fp, key, tos) {
			if (fi->fib_priority != FIB_INFO(f)->fib_priority)
				break;
			if (f->fn_type == type && f->fn_scope == r->rtm_scope
			    && FIB_INFO(f) == fi)
				goto out;
		}

		if (!(n->nlmsg_flags&NLM_F_APPEND)) {
			fp = ins_fp;
			f = *fp;
		}
	}

create:
	err = -ENOENT;
	if (!(n->nlmsg_flags&NLM_F_CREATE))
		goto out;

replace:
	err = -ENOBUFS;
	new_f = kmem_cache_alloc(fn_hash_kmem, SLAB_KERNEL);
	if (new_f == NULL)
		goto out;

	memset(new_f, 0, sizeof(struct fib_node));

	new_f->fn_key = key;
#ifdef CONFIG_IP_ROUTE_TOS
	new_f->fn_tos = tos;
#endif
	new_f->fn_type = type;
	new_f->fn_scope = r->rtm_scope;
	FIB_INFO(new_f) = fi;

	/*
	 * Insert new entry to the list.
	 */

	new_f->fn_next = f;
	write_lock_bh(&fib_hash_lock);
	*fp = new_f;
	write_unlock_bh(&fib_hash_lock);
	fz->fz_nent++;

	if (del_fp) {
		f = *del_fp;
		/* Unlink replaced node */
		write_lock_bh(&fib_hash_lock);
		*del_fp = f->fn_next;
		write_unlock_bh(&fib_hash_lock);

		if (!(f->fn_state&FN_S_ZOMBIE))
			rtmsg_fib(RTM_DELROUTE, f, z, tb->tb_id, n, req);
		if (f->fn_state&FN_S_ACCESSED)
			rt_cache_flush(-1);
		fn_free_node(f);
		fz->fz_nent--;
	} else {
		rt_cache_flush(-1);
	}
	rtmsg_fib(RTM_NEWROUTE, new_f, z, tb->tb_id, n, req);
	return 0;

out:
	fib_release_info(fi);
	return err;
}


static int
fn_hash_delete(struct fib_table *tb, struct rtmsg *r, struct kern_rta *rta,
		struct nlmsghdr *n, struct netlink_skb_parms *req)
{
	struct fn_hash *table = (struct fn_hash*)tb->tb_data;
	struct fib_node **fp, **del_fp, *f;
	int z = r->rtm_dst_len;
	struct fn_zone *fz;
	fn_key_t key;
	int matched;
#ifdef CONFIG_IP_ROUTE_TOS
	u8 tos = r->rtm_tos;
#endif

FTprint("tb(%d)_delete: %d %08x/%d %d\n", tb->tb_id, r->rtm_type, rta->rta_dst ?
       *(u32*)rta->rta_dst : 0, z, rta->rta_oif ? *rta->rta_oif : -1);
	if (z > 32)
		return -EINVAL;
	if ((fz  = table->fn_zones[z]) == NULL)
		return -ESRCH;

	fz_key_0(key);
	if (rta->rta_dst) {
		u32 dst;
		memcpy(&dst, rta->rta_dst, 4);
		if (dst & ~FZ_MASK(fz))
			return -EINVAL;
		key = fz_key(dst, fz);
	}

	fp = fz_chain_p(key, fz);


	FIB_SCAN(f, fp) {
		if (fn_key_eq(f->fn_key, key))
			break;
		if (fn_key_leq(key, f->fn_key)) {
			return -ESRCH;
		}
	}
#ifdef CONFIG_IP_ROUTE_TOS
	FIB_SCAN_KEY(f, fp, key) {
		if (f->fn_tos == tos)
			break;
	}
#endif

	matched = 0;
	del_fp = NULL;
	FIB_SCAN_TOS(f, fp, key, tos) {
		struct fib_info * fi = FIB_INFO(f);

		if (f->fn_state&FN_S_ZOMBIE) {
			return -ESRCH;
		}
		matched++;

		if (del_fp == NULL &&
		    (!r->rtm_type || f->fn_type == r->rtm_type) &&
		    (r->rtm_scope == RT_SCOPE_NOWHERE || f->fn_scope == r->rtm_scope) &&
		    (!r->rtm_protocol || fi->fib_protocol == r->rtm_protocol) &&
		    fib_nh_match(r, n, rta, fi) == 0)
			del_fp = fp;
	}

	if (del_fp) {
		f = *del_fp;
		rtmsg_fib(RTM_DELROUTE, f, z, tb->tb_id, n, req);

		if (matched != 1) {
			write_lock_bh(&fib_hash_lock);
			*del_fp = f->fn_next;
			write_unlock_bh(&fib_hash_lock);

			if (f->fn_state&FN_S_ACCESSED)
				rt_cache_flush(-1);
			fn_free_node(f);
			fz->fz_nent--;
		} else {
			f->fn_state |= FN_S_ZOMBIE;
			if (f->fn_state&FN_S_ACCESSED) {
				f->fn_state &= ~FN_S_ACCESSED;
				rt_cache_flush(-1);
			}
			if (++fib_hash_zombies > 128)
				fib_flush();
		}

		return 0;
	}
	return -ESRCH;
}

static __inline__ int
fn_flush_list(struct fib_node ** fp, int z, struct fn_hash *table)
{
	int found = 0;
	struct fib_node *f;

	while ((f = *fp) != NULL) {
		struct fib_info *fi = FIB_INFO(f);

		if (fi && ((f->fn_state&FN_S_ZOMBIE) || (fi->fib_flags&RTNH_F_DEAD))) {
			write_lock_bh(&fib_hash_lock);
			*fp = f->fn_next;
			write_unlock_bh(&fib_hash_lock);

			fn_free_node(f);
			found++;
			continue;
		}
		fp = &f->fn_next;
	}
	return found;
}

static int fn_hash_flush(struct fib_table *tb)
{
	struct fn_hash *table = (struct fn_hash*)tb->tb_data;
	struct fn_zone *fz;
	int found = 0;

	fib_hash_zombies = 0;
	for (fz = table->fn_zone_list; fz; fz = fz->fz_next) {
		int i;
		int tmp = 0;
		for (i=fz->fz_divisor-1; i>=0; i--)
			tmp += fn_flush_list(&fz->fz_hash[i], fz->fz_order, table);
		fz->fz_nent -= tmp;
		found += tmp;
	}
	return found;
}


static __inline__ int
fn_hash_dump_bucket(struct sk_buff *skb, struct netlink_callback *cb,
		     struct fib_table *tb,
		     struct fn_zone *fz,
		     struct fib_node *f)
{
	int i, s_i;

	s_i = cb->args[3];
	for (i=0; f; i++, f=f->fn_next) {
		if (i < s_i) continue;
		if (f->fn_state&FN_S_ZOMBIE) continue;
		if (fib_dump_info(skb, NETLINK_CB(cb->skb).pid, cb->nlh->nlmsg_seq,
				  RTM_NEWROUTE,
				  tb->tb_id, (f->fn_state&FN_S_ZOMBIE) ? 0 : f->fn_type, f->fn_scope,
				  &f->fn_key, fz->fz_order, f->fn_tos,
				  f->fn_info) < 0) {
			cb->args[3] = i;
			return -1;
		}
	}
	cb->args[3] = i;
	return skb->len;
}

static __inline__ int
fn_hash_dump_zone(struct sk_buff *skb, struct netlink_callback *cb,
		   struct fib_table *tb,
		   struct fn_zone *fz)
{
	int h, s_h;

	s_h = cb->args[2];
	for (h=0; h < fz->fz_divisor; h++) {
		if (h < s_h) continue;
		if (h > s_h)
			memset(&cb->args[3], 0, sizeof(cb->args) - 3*sizeof(cb->args[0]));
		if (fz->fz_hash == NULL || fz->fz_hash[h] == NULL)
			continue;
		if (fn_hash_dump_bucket(skb, cb, tb, fz, fz->fz_hash[h]) < 0) {
			cb->args[2] = h;
			return -1;
		}
	}
	cb->args[2] = h;
	return skb->len;
}

static int fn_hash_dump(struct fib_table *tb, struct sk_buff *skb, struct netlink_callback *cb)
{
	int m, s_m;
	struct fn_zone *fz;
	struct fn_hash *table = (struct fn_hash*)tb->tb_data;

	s_m = cb->args[1];
	read_lock(&fib_hash_lock);
	for (fz = table->fn_zone_list, m=0; fz; fz = fz->fz_next, m++) {
		if (m < s_m) continue;
		if (m > s_m)
			memset(&cb->args[2], 0, sizeof(cb->args) - 2*sizeof(cb->args[0]));
		if (fn_hash_dump_zone(skb, cb, tb, fz) < 0) {
			cb->args[1] = m;
			read_unlock(&fib_hash_lock);
			return -1;
		}
	}
	read_unlock(&fib_hash_lock);
	cb->args[1] = m;
	return skb->len;
}

static void rtmsg_fib(int event, struct fib_node* f, int z, int tb_id,
		      struct nlmsghdr *n, struct netlink_skb_parms *req)
{
	struct sk_buff *skb;
	u32 pid = req ? req->pid : 0;
	int size = NLMSG_SPACE(sizeof(struct rtmsg)+256);

	skb = alloc_skb(size, GFP_KERNEL);
	if (!skb)
		return;

	if (fib_dump_info(skb, pid, n->nlmsg_seq, event, tb_id,
			  f->fn_type, f->fn_scope, &f->fn_key, z, f->fn_tos,
			  FIB_INFO(f)) < 0) {
		kfree_skb(skb);
		return;
	}
	NETLINK_CB(skb).dst_groups = RTMGRP_IPV4_ROUTE;
	if (n->nlmsg_flags&NLM_F_ECHO)
		atomic_inc(&skb->users);
	netlink_broadcast(rtnl, skb, pid, RTMGRP_IPV4_ROUTE, GFP_KERNEL);
	if (n->nlmsg_flags&NLM_F_ECHO)
		netlink_unicast(rtnl, skb, pid, MSG_DONTWAIT);
}

#ifdef CONFIG_IP_MULTIPLE_TABLES
struct fib_table * fib_hash_init(int id)
#else
struct fib_table * __init fib_hash_init(int id)
#endif
{
	struct fib_table *tb;

	if (fn_hash_kmem == NULL)
		fn_hash_kmem = kmem_cache_create("ip_fib_hash",
						 sizeof(struct fib_node),
						 0, SLAB_HWCACHE_ALIGN,
						 NULL, NULL);

	tb = kmalloc(sizeof(struct fib_table) + sizeof(struct fn_hash), GFP_KERNEL);
	if (tb == NULL)
		return NULL;

	tb->tb_id = id;
	tb->tb_lookup = fn_hash_lookup;
	tb->tb_insert = fn_hash_insert;
	tb->tb_delete = fn_hash_delete;
	tb->tb_flush = fn_hash_flush;
	tb->tb_select_default = fn_hash_select_default;
	tb->tb_dump = fn_hash_dump;
	memset(tb->tb_data, 0, sizeof(struct fn_hash));
	return tb;
}

/* ------------------------------------------------------------------------ */
#ifdef CONFIG_PROC_FS

struct fib_iter_state {
	struct fn_zone	*zone;
	int		bucket;
	struct fib_node **hash;
	struct fib_node *node;
};

static __inline__ struct fib_node *fib_get_first(struct seq_file *seq)
{
	struct fib_iter_state* iter = seq->private;
	struct fn_hash *table = (struct fn_hash *)ip_fib_main_table->tb_data;

	iter->bucket = 0;
	iter->hash   = NULL;
	iter->node   = NULL;

	for (iter->zone = table->fn_zone_list; iter->zone;
	     iter->zone = iter->zone->fz_next) {
		int maxslot;

		if (!iter->zone->fz_next)
			continue;

		iter->hash = iter->zone->fz_hash;
		maxslot = iter->zone->fz_divisor;

		for (iter->bucket = 0; iter->bucket < maxslot;
		     ++iter->bucket, ++iter->hash) {
			iter->node = *iter->hash;

			if (iter->node)
				goto out;
		}
	}
out:
	return iter->node;
}

static __inline__ struct fib_node *fib_get_next(struct seq_file *seq)
{
	struct fib_iter_state* iter = seq->private;

	if (iter->node)
		iter->node = iter->node->fn_next;

	if (iter->node)
		goto out;

	if (!iter->zone)
		goto out;

	for (;;) {
		int maxslot;

		maxslot = iter->zone->fz_divisor;

		while (++iter->bucket < maxslot) {
			iter->node = *++iter->hash;

			if (iter->node)
				goto out;
		}

		iter->zone = iter->zone->fz_next;

		if (!iter->zone)
			goto out;
		
		iter->hash = iter->zone->fz_hash;
		iter->bucket = 0;
		iter->node = *iter->hash;
		if (iter->node)
			break;
	}
out:
	return iter->node;
}

static void *fib_seq_start(struct seq_file *seq, loff_t *pos)
{
	void *v = NULL;

	read_lock(&fib_hash_lock);
	if (ip_fib_main_table)
		v = *pos ? fib_get_next(seq) : SEQ_START_TOKEN;
	return v;
}

static void *fib_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	++*pos;
	return v == SEQ_START_TOKEN ? fib_get_first(seq) : fib_get_next(seq);
}

static void fib_seq_stop(struct seq_file *seq, void *v)
{
	read_unlock(&fib_hash_lock);
}

static unsigned fib_flag_trans(int type, int dead, u32 mask, struct fib_info *fi)
{
	static unsigned type2flags[RTN_MAX + 1] = {
		[7] = RTF_REJECT, [8] = RTF_REJECT,
	};
	unsigned flags = type2flags[type];

	if (fi && fi->fib_nh->nh_gw)
		flags |= RTF_GATEWAY;
	if (mask == 0xFFFFFFFF)
		flags |= RTF_HOST;
	if (!dead)
		flags |= RTF_UP;
	return flags;
}

/* 
 *	This outputs /proc/net/route.
 *
 *	It always works in backward compatibility mode.
 *	The format of the file is not supposed to be changed.
 */
static int fib_seq_show(struct seq_file *seq, void *v)
{
	struct fib_iter_state* iter;
	char bf[128];
	u32 prefix, mask;
	unsigned flags;
	struct fib_node *f;
	struct fib_info *fi;

	if (v == SEQ_START_TOKEN) {
		seq_printf(seq, "%-127s\n", "Iface\tDestination\tGateway "
			   "\tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU"
			   "\tWindow\tIRTT");
		goto out;
	}

	f	= v;
	fi	= FIB_INFO(f);
	iter	= seq->private;
	prefix	= fz_prefix(f->fn_key, iter->zone);
	mask	= FZ_MASK(iter->zone);
	flags	= fib_flag_trans(f->fn_type, f->fn_state & FN_S_ZOMBIE,
				 mask, fi);
	if (fi)
		snprintf(bf, sizeof(bf),
			 "%s\t%08X\t%08X\t%04X\t%d\t%u\t%d\t%08X\t%d\t%u\t%u",
			 fi->fib_dev ? fi->fib_dev->name : "*", prefix,
			 fi->fib_nh->nh_gw, flags, 0, 0, fi->fib_priority,
			 mask, (fi->fib_advmss ? fi->fib_advmss + 40 : 0),
			 fi->fib_window,
			 fi->fib_rtt >> 3);
	else
		snprintf(bf, sizeof(bf),
			 "*\t%08X\t%08X\t%04X\t%d\t%u\t%d\t%08X\t%d\t%u\t%u",
			 prefix, 0, flags, 0, 0, 0, mask, 0, 0, 0);
	seq_printf(seq, "%-127s\n", bf);
out:
	return 0;
}

static struct seq_operations fib_seq_ops = {
	.start  = fib_seq_start,
	.next   = fib_seq_next,
	.stop   = fib_seq_stop,
	.show   = fib_seq_show,
};

static int fib_seq_open(struct inode *inode, struct file *file)
{
	struct seq_file *seq;
	int rc = -ENOMEM;
	struct fib_iter_state *s = kmalloc(sizeof(*s), GFP_KERNEL);
       
	if (!s)
		goto out;

	rc = seq_open(file, &fib_seq_ops);
	if (rc)
		goto out_kfree;

	seq	     = file->private_data;
	seq->private = s;
	memset(s, 0, sizeof(*s));
out:
	return rc;
out_kfree:
	kfree(s);
	goto out;
}

static struct file_operations fib_seq_fops = {
	.owner		= THIS_MODULE,
	.open           = fib_seq_open,
	.read           = seq_read,
	.llseek         = seq_lseek,
	.release	= seq_release_private,
};

int __init fib_proc_init(void)
{
	if (!proc_net_fops_create("route", S_IRUGO, &fib_seq_fops))
		return -ENOMEM;
	return 0;
}

void __init fib_proc_exit(void)
{
	proc_net_remove("route");
}
#endif /* CONFIG_PROC_FS */