- Remove rt_metrics_lite and simply put its members into rtentry.

- Use counter(9) for rt_pksent (former rt_rmx.rmx_pksent). This
  removes another cache trashing ++ from packet forwarding path.
- Create zini/fini methods for the rtentry UMA zone. Via initialize
  mutex and counter in them.
- Fix reporting of rmx_pksent to routing socket.
- Fix netstat(1) to report "Use" both in kvm(3) and sysctl(3) mode.

The change is mostly targeted for stable/10 merge. For head,
rt_pksent is expected to just disappear.

Discussed with:		melifaro
Sponsored by:		Netflix
Sponsored by:		Nginx, Inc.

share/man/man9/rtentry.9

@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 16, 2013
+.Dd March 5, 2014
 .Dt RTENTRY 9
 .Os
 .Sh NAME
@@ -78,6 +78,12 @@ intermediate stop on the way to that destination (if the
 flag is set).
 .It Vt "int rt_flags" ;
 See below.
+If the
+.Dv RTF_UP
+flag is not present, the
+.Fn rtfree
+function will delete the route from the radix tree when the last
+reference drops.
 .It Vt "int rt_refcnt" ;
 Route entries are reference-counted; this field indicates the number
 of external (to the radix tree) references.
@@ -89,14 +95,14 @@ as it were, to the question posed by a route lookup; that is, they
 name the interface and interface address to be used in sending a
 packet to the destination or set of destinations which this route
 represents.
-.It Vt "struct rt_metrics_lite rt_rmx" ;
-See below.
-If the
-.Dv RTF_UP
-flag is not present, the
-.Fn rtfree
-function will delete the route from the radix tree when the last
-reference drops.
+.It Vt "u_long rt_mtu";
+See description of rmx_mtu below.
+.It Vt "u_long rt_weight";
+See description of rmx_weight below.
+.It Vt "u_long rt_expire";
+See description of rmx_expire below.
+.It Vt "counter64_t rt_pksent";
+See description of rmx_pksent below.
 .It Vt "struct rtentry *rt_gwroute" ;
 This member is a reference to a route whose destination is
 .Va rt_gateway .
@@ -164,9 +170,7 @@ Indicates that the destination is a broadcast address.
 Indicates that the destination is a multicast address.
 .El
 .Pp
-Every route has associated with it a set of metrics, stored in
-.Vt "struct rt_metrics_lite" .
-Metrics are supplied in
+Several metrics are supplied in
 .Vt "struct rt_metrics"
 passed with routing control messages via
 .Xr route 4
@@ -175,8 +179,7 @@ Currently only
 .Vt rmx_mtu , rmx_expire ,
 and
 .Vt rmx_pksent
-metrics are used in
-.Vt "struct rt_metrics_lite" .
+metrics are supplied.
 All others are ignored.
 .Pp
 The following metrics are defined by

sys/contrib/ipfilter/netinet/ip_fil_freebsd.c

@@ -802,7 +802,7 @@ ipf_fastroute(m0, mpp, fin, fdp)
 	if (ro->ro_rt->rt_flags & RTF_GATEWAY)
 		dst = (struct sockaddr_in *)ro->ro_rt->rt_gateway;
 	if (ro->ro_rt)
-		ro->ro_rt->rt_use++;
+		counter_u64_add(ro->ro_rt->rt_pksent, 1);
 
 	/*
 	 * For input packets which are being "fastrouted", they won't

sys/net/if_disc.c

@@ -186,7 +186,7 @@ static void
 discrtrequest(int cmd, struct rtentry *rt, struct rt_addrinfo *info)
 {
 	RT_LOCK_ASSERT(rt);
-	rt->rt_rmx.rmx_mtu = DSMTU;
+	rt->rt_mtu = DSMTU;
 }
 
 /*

sys/net/if_faith.c

@@ -246,7 +246,7 @@ faithrtrequest(cmd, rt, info)
 	struct rt_addrinfo *info;
 {
 	RT_LOCK_ASSERT(rt);
-	rt->rt_rmx.rmx_mtu = rt->rt_ifp->if_mtu;
+	rt->rt_mtu = rt->rt_ifp->if_mtu;
 }
 
 /*

sys/net/if_loop.c

@@ -395,7 +395,7 @@ lortrequest(int cmd, struct rtentry *rt, struct rt_addrinfo *info)
 {
 
 	RT_LOCK_ASSERT(rt);
-	rt->rt_rmx.rmx_mtu = rt->rt_ifp->if_mtu;
+	rt->rt_mtu = rt->rt_ifp->if_mtu;
 }
 
 /*

sys/net/if_stf.c

@@ -785,7 +785,7 @@ stf_rtrequest(cmd, rt, info)
 	struct rt_addrinfo *info;
 {
 	RT_LOCK_ASSERT(rt);
-	rt->rt_rmx.rmx_mtu = rt->rt_ifp->if_mtu;
+	rt->rt_mtu = rt->rt_ifp->if_mtu;
 }
 
 static int

sys/net/radix_mpath.c

@@ -85,7 +85,7 @@ rn_mpath_count(struct radix_node *rn)
 	
 	while (rn != NULL) {
 		rt = (struct rtentry *)rn;
-		i += rt->rt_rmx.rmx_weight;
+		i += rt->rt_weight;
 		rn = rn_mpath_next(rn);
 	}
 	return (i);
@@ -230,8 +230,8 @@ rtalloc_mpath_fib(struct route *ro, uint32_t hash, u_int fibnum)
 	hash += hashjitter;
 	hash %= n;
 	for (weight = abs((int32_t)hash), rt = ro->ro_rt;
-	     weight >= rt->rt_rmx.rmx_weight && rn; 
-	     weight -= rt->rt_rmx.rmx_weight) {
+	     weight >= rt->rt_weight && rn; 
+	     weight -= rt->rt_weight) {
 		
 		/* stay within the multipath routes */
 		if (rn->rn_dupedkey && rn->rn_mask != rn->rn_dupedkey->rn_mask)

sys/net/route.c

@@ -202,6 +202,40 @@ route_init(void)
 }
 SYSINIT(route_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, route_init, 0);
 
+static int
+rtentry_zinit(void *mem, int size, int how)
+{
+	struct rtentry *rt = mem;
+
+	rt->rt_pksent = counter_u64_alloc(how);
+	if (rt->rt_pksent == NULL)
+		return (ENOMEM);
+
+	RT_LOCK_INIT(rt);
+
+	return (0);
+}
+
+static void
+rtentry_zfini(void *mem, int size)
+{
+	struct rtentry *rt = mem;
+
+	RT_LOCK_DESTROY(rt);
+	counter_u64_free(rt->rt_pksent);
+}
+
+static int
+rtentry_ctor(void *mem, int size, void *arg, int how)
+{
+	struct rtentry *rt = mem;
+
+	bzero(rt, offsetof(struct rtentry, rt_endzero));
+	counter_u64_zero(rt->rt_pksent);
+
+	return (0);
+}
+
 static void
 vnet_route_init(const void *unused __unused)
 {
@@ -213,8 +247,9 @@ vnet_route_init(const void *unused __unused)
 	V_rt_tables = malloc(rt_numfibs * (AF_MAX+1) *
 	    sizeof(struct radix_node_head *), M_RTABLE, M_WAITOK|M_ZERO);
 
-	V_rtzone = uma_zcreate("rtentry", sizeof(struct rtentry), NULL, NULL,
-	    NULL, NULL, UMA_ALIGN_PTR, 0);
+	V_rtzone = uma_zcreate("rtentry", sizeof(struct rtentry),
+	    rtentry_ctor, NULL,
+	    rtentry_zinit, rtentry_zfini, UMA_ALIGN_PTR, 0);
 	for (dom = domains; dom; dom = dom->dom_next) {
 		if (dom->dom_rtattach == NULL)
 			continue;
@@ -491,7 +526,6 @@ rtfree(struct rtentry *rt)
 		/*
 		 * and the rtentry itself of course
 		 */
-		RT_LOCK_DESTROY(rt);
 		uma_zfree(V_rtzone, rt);
 		return;
 	}
@@ -1227,12 +1261,11 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 		} else
 			ifa_ref(info->rti_ifa);
 		ifa = info->rti_ifa;
-		rt = uma_zalloc(V_rtzone, M_NOWAIT | M_ZERO);
+		rt = uma_zalloc(V_rtzone, M_NOWAIT);
 		if (rt == NULL) {
 			ifa_free(ifa);
 			senderr(ENOBUFS);
 		}
-		RT_LOCK_INIT(rt);
 		rt->rt_flags = RTF_UP | flags;
 		rt->rt_fibnum = fibnum;
 		/*
@@ -1240,7 +1273,6 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 		 */
 		RT_LOCK(rt);
 		if ((error = rt_setgate(rt, dst, gateway)) != 0) {
-			RT_LOCK_DESTROY(rt);
 			ifa_free(ifa);
 			uma_zfree(V_rtzone, rt);
 			senderr(error);
@@ -1266,7 +1298,7 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 		 */
 		rt->rt_ifa = ifa;
 		rt->rt_ifp = ifa->ifa_ifp;
-		rt->rt_rmx.rmx_weight = 1;
+		rt->rt_weight = 1;
 
 #ifdef RADIX_MPATH
 		/* do not permit exactly the same dst/mask/gw pair */
@@ -1274,7 +1306,6 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 			rt_mpath_conflict(rnh, rt, netmask)) {
 			ifa_free(rt->rt_ifa);
 			Free(rt_key(rt));
-			RT_LOCK_DESTROY(rt);
 			uma_zfree(V_rtzone, rt);
 			senderr(EEXIST);
 		}
@@ -1342,7 +1373,6 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 		if (rn == NULL) {
 			ifa_free(rt->rt_ifa);
 			Free(rt_key(rt));
-			RT_LOCK_DESTROY(rt);
 			uma_zfree(V_rtzone, rt);
 #ifdef FLOWTABLE
 			if (rt0 != NULL)

sys/net/route.h

@@ -33,6 +33,8 @@
 #ifndef _NET_ROUTE_H_
 #define _NET_ROUTE_H_
 
+#include <sys/counter.h>
+
 /*
  * Kernel resident routing tables.
  *
@@ -57,17 +59,6 @@ struct route {
 #define	RT_CACHING_CONTEXT	0x1	/* XXX: not used anywhere */
 #define	RT_NORTREF		0x2	/* doesn't hold reference on ro_rt */
 
-/*
- * These numbers are used by reliable protocols for determining
- * retransmission behavior and are included in the routing structure.
- */
-struct rt_metrics_lite {
-	u_long	rmx_mtu;	/* MTU for this path */
-	u_long	rmx_expire;	/* lifetime for route, e.g. redirect */
-	u_long	rmx_pksent;	/* packets sent using this route */
-	u_long	rmx_weight;	/* absolute weight */ 
-};
-
 struct rt_metrics {
 	u_long	rmx_locks;	/* Kernel must leave these values alone */
 	u_long	rmx_mtu;	/* MTU for this path */
@@ -124,16 +115,17 @@ struct rtentry {
 #define	rt_key(r)	(*((struct sockaddr **)(&(r)->rt_nodes->rn_key)))
 #define	rt_mask(r)	(*((struct sockaddr **)(&(r)->rt_nodes->rn_mask)))
 	struct	sockaddr *rt_gateway;	/* value */
-	int	rt_flags;		/* up/down?, host/net */
-	int	rt_refcnt;		/* # held references */
 	struct	ifnet *rt_ifp;		/* the answer: interface to use */
 	struct	ifaddr *rt_ifa;		/* the answer: interface address to use */
-	struct	rt_metrics_lite rt_rmx;	/* metrics used by rx'ing protocols */
-	u_int	rt_fibnum;		/* which FIB */
-#ifdef _KERNEL
-	/* XXX ugly, user apps use this definition but don't have a mtx def */
-	struct	mtx rt_mtx;		/* mutex for routing entry */
-#endif
+	int		rt_flags;	/* up/down?, host/net */
+	int		rt_refcnt;	/* # held references */
+	u_int		rt_fibnum;	/* which FIB */
+	u_long		rt_mtu;		/* MTU for this path */
+	u_long		rt_weight;	/* absolute weight */ 
+	u_long		rt_expire;	/* lifetime for route, e.g. redirect */
+#define	rt_endzero	rt_pksent
+	counter_u64_t	rt_pksent;	/* packets sent using this route */
+	struct mtx	rt_mtx;		/* mutex for routing entry */
 };
 
 /*
@@ -150,8 +142,6 @@ struct ortentry {
 	struct	ifnet *rt_ifp;		/* the answer: interface to use */
 };
 
-#define rt_use rt_rmx.rmx_pksent
-
 #define	RTF_UP		0x1		/* route usable */
 #define	RTF_GATEWAY	0x2		/* destination is a gateway */
 #define	RTF_HOST	0x4		/* host entry (net otherwise) */

sys/net/rtsock.c

@@ -189,10 +189,8 @@ static int	sysctl_dumpentry(struct radix_node *rn, void *vw);
 static int	sysctl_iflist(int af, struct walkarg *w);
 static int	sysctl_ifmalist(int af, struct walkarg *w);
 static int	route_output(struct mbuf *m, struct socket *so);
-static void	rt_setmetrics(u_long which, const struct rt_metrics *in,
-			struct rt_metrics_lite *out);
-static void	rt_getmetrics(const struct rt_metrics_lite *in,
-			struct rt_metrics *out);
+static void	rt_setmetrics(const struct rt_msghdr *rtm, struct rtentry *rt);
+static void	rt_getmetrics(const struct rtentry *rt, struct rt_metrics *out);
 static void	rt_dispatch(struct mbuf *, sa_family_t);
 
 static struct netisr_handler rtsock_nh = {
@@ -680,8 +678,7 @@ route_output(struct mbuf *m, struct socket *so)
 			rti_need_deembed = (V_deembed_scopeid) ? 1 : 0;
 #endif
 			RT_LOCK(saved_nrt);
-			rt_setmetrics(rtm->rtm_inits,
-				&rtm->rtm_rmx, &saved_nrt->rt_rmx);
+			rt_setmetrics(rtm, saved_nrt);
 			rtm->rtm_index = saved_nrt->rt_ifp->if_index;
 			RT_REMREF(saved_nrt);
 			RT_UNLOCK(saved_nrt);
@@ -850,7 +847,7 @@ route_output(struct mbuf *m, struct socket *so)
 					(rt->rt_flags & ~RTF_GWFLAG_COMPAT);
 			else
 				rtm->rtm_flags = rt->rt_flags;
-			rt_getmetrics(&rt->rt_rmx, &rtm->rtm_rmx);
+			rt_getmetrics(rt, &rtm->rtm_rmx);
 			rtm->rtm_addrs = info.rti_addrs;
 			break;
 
@@ -913,8 +910,7 @@ route_output(struct mbuf *m, struct socket *so)
 			/* Allow some flags to be toggled on change. */
 			rt->rt_flags = (rt->rt_flags & ~RTF_FMASK) |
 				    (rtm->rtm_flags & RTF_FMASK);
-			rt_setmetrics(rtm->rtm_inits, &rtm->rtm_rmx,
-					&rt->rt_rmx);
+			rt_setmetrics(rtm, rt);
 			rtm->rtm_index = rt->rt_ifp->if_index;
 			if (rt->rt_ifa && rt->rt_ifa->ifa_rtrequest)
 			       rt->rt_ifa->ifa_rtrequest(RTM_ADD, rt, &info);
@@ -1002,34 +998,30 @@ flush:
 }
 
 static void
-rt_setmetrics(u_long which, const struct rt_metrics *in,
-	struct rt_metrics_lite *out)
+rt_setmetrics(const struct rt_msghdr *rtm, struct rtentry *rt)
 {
-#define metric(f, e) if (which & (f)) out->e = in->e;
-	/*
-	 * Only these are stored in the routing entry since introduction
-	 * of tcp hostcache. The rest is ignored.
-	 */
-	metric(RTV_MTU, rmx_mtu);
-	metric(RTV_WEIGHT, rmx_weight);
-	/* Userland -> kernel timebase conversion. */
-	if (which & RTV_EXPIRE)
-		out->rmx_expire = in->rmx_expire ?
-		    in->rmx_expire - time_second + time_uptime : 0;
-#undef metric
+
+	if (rtm->rtm_inits & RTV_MTU)
+		rt->rt_mtu = rtm->rtm_rmx.rmx_mtu;
+	if (rtm->rtm_inits & RTV_WEIGHT)
+		rt->rt_weight = rtm->rtm_rmx.rmx_weight;
+	/* Kernel -> userland timebase conversion. */
+	if (rtm->rtm_inits & RTV_EXPIRE)
+		rt->rt_expire = rtm->rtm_rmx.rmx_expire ?
+		    rtm->rtm_rmx.rmx_expire - time_second + time_uptime : 0;
 }
 
 static void
-rt_getmetrics(const struct rt_metrics_lite *in, struct rt_metrics *out)
+rt_getmetrics(const struct rtentry *rt, struct rt_metrics *out)
 {
-#define metric(e) out->e = in->e;
+
 	bzero(out, sizeof(*out));
-	metric(rmx_mtu);
-	metric(rmx_weight);
+	out->rmx_mtu = rt->rt_mtu;
+	out->rmx_weight = rt->rt_weight;
+	out->rmx_pksent = counter_u64_fetch(rt->rt_pksent);
 	/* Kernel -> userland timebase conversion. */
-	out->rmx_expire = in->rmx_expire ?
-	    in->rmx_expire - time_uptime + time_second : 0;
-#undef metric
+	out->rmx_expire = rt->rt_expire ?
+	    rt->rt_expire - time_uptime + time_second : 0;
 }
 
 /*
@@ -1596,11 +1588,7 @@ sysctl_dumpentry(struct radix_node *rn, void *vw)
 				(rt->rt_flags & ~RTF_GWFLAG_COMPAT);
 		else
 			rtm->rtm_flags = rt->rt_flags;
-		/*
-		 * let's be honest about this being a retarded hack
-		 */
-		rtm->rtm_fmask = rt->rt_rmx.rmx_pksent;
-		rt_getmetrics(&rt->rt_rmx, &rtm->rtm_rmx);
+		rt_getmetrics(rt, &rtm->rtm_rmx);
 		rtm->rtm_index = rt->rt_ifp->if_index;
 		rtm->rtm_errno = rtm->rtm_pid = rtm->rtm_seq = 0;
 		rtm->rtm_addrs = info.rti_addrs;

sys/netatalk/ddp_output.c

@@ -222,7 +222,7 @@ ddp_route(struct mbuf *m, struct route *ro)
 		elh->el_type = ELAP_DDPEXTEND;
 		elh->el_dnode = gate.sat_addr.s_node;
 	}
-	ro->ro_rt->rt_use++;
+	counter_u64_add(ro->ro_rt->rt_pksent, 1);
 
 #ifdef NETATALK_DEBUG
 	printf ("ddp_route: from %d.%d to %d.%d, via %d.%d (%s)\n",

sys/netinet/in_rmx.c

@@ -94,8 +94,8 @@ in_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
 	if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr)))
 		rt->rt_flags |= RTF_MULTICAST;
 
-	if (!rt->rt_rmx.rmx_mtu && rt->rt_ifp)
-		rt->rt_rmx.rmx_mtu = rt->rt_ifp->if_mtu;
+	if (rt->rt_mtu == 0 && rt->rt_ifp != NULL)
+		rt->rt_mtu = rt->rt_ifp->if_mtu;
 
 	return (rn_addroute(v_arg, n_arg, head, treenodes));
 }
@@ -115,7 +115,7 @@ in_matroute(void *v_arg, struct radix_node_head *head)
 		RT_LOCK(rt);
 		if (rt->rt_flags & RTPRF_OURS) {
 			rt->rt_flags &= ~RTPRF_OURS;
-			rt->rt_rmx.rmx_expire = 0;
+			rt->rt_expire = 0;
 		}
 		RT_UNLOCK(rt);
 	}
@@ -168,7 +168,7 @@ in_clsroute(struct radix_node *rn, struct radix_node_head *head)
 	 */
 	if (V_rtq_reallyold != 0) {
 		rt->rt_flags |= RTPRF_OURS;
-		rt->rt_rmx.rmx_expire = time_uptime + V_rtq_reallyold;
+		rt->rt_expire = time_uptime + V_rtq_reallyold;
 	} else {
 		rtexpunge(rt);
 	}
@@ -200,7 +200,7 @@ in_rtqkill(struct radix_node *rn, void *rock)
 	if (rt->rt_flags & RTPRF_OURS) {
 		ap->found++;
 
-		if (ap->draining || rt->rt_rmx.rmx_expire <= time_uptime) {
+		if (ap->draining || rt->rt_expire <= time_uptime) {
 			if (rt->rt_refcnt > 0)
 				panic("rtqkill route really not free");
 
@@ -216,13 +216,9 @@ in_rtqkill(struct radix_node *rn, void *rock)
 			}
 		} else {
 			if (ap->updating &&
-			    (rt->rt_rmx.rmx_expire - time_uptime >
-			     V_rtq_reallyold)) {
-				rt->rt_rmx.rmx_expire =
-				    time_uptime + V_rtq_reallyold;
-			}
-			ap->nextstop = lmin(ap->nextstop,
-					    rt->rt_rmx.rmx_expire);
+			    (rt->rt_expire - time_uptime > V_rtq_reallyold))
+				rt->rt_expire = time_uptime + V_rtq_reallyold;
+			ap->nextstop = lmin(ap->nextstop, rt->rt_expire);
 		}
 	}
 

sys/netinet/ip_fastfwd.c

@@ -490,8 +490,7 @@ passout:
 	 * Check if route is dampned (when ARP is unable to resolve)
 	 */
 	if ((ro.ro_rt->rt_flags & RTF_REJECT) &&
-	    (ro.ro_rt->rt_rmx.rmx_expire == 0 ||
-	    time_uptime < ro.ro_rt->rt_rmx.rmx_expire)) {
+	    (ro.ro_rt->rt_expire == 0 || time_uptime < ro.ro_rt->rt_expire)) {
 		icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0);
 		goto consumed;
 	}
@@ -519,8 +518,8 @@ passout:
 	/*
 	 * Check if packet fits MTU or if hardware will fragment for us
 	 */
-	if (ro.ro_rt->rt_rmx.rmx_mtu)
-		mtu = min(ro.ro_rt->rt_rmx.rmx_mtu, ifp->if_mtu);
+	if (ro.ro_rt->rt_mtu)
+		mtu = min(ro.ro_rt->rt_mtu, ifp->if_mtu);
 	else
 		mtu = ifp->if_mtu;
 
@@ -585,7 +584,7 @@ passout:
 	if (error != 0)
 		IPSTAT_INC(ips_odropped);
 	else {
-		ro.ro_rt->rt_rmx.rmx_pksent++;
+		counter_u64_add(ro.ro_rt->rt_pksent, 1);
 		IPSTAT_INC(ips_forward);
 		IPSTAT_INC(ips_fastforward);
 	}

sys/netinet/ip_input.c

@@ -1463,7 +1463,7 @@ ip_forward(struct mbuf *m, int srcrt)
 	error = ip_output(m, NULL, &ro, IP_FORWARDING, NULL, NULL);
 
 	if (error == EMSGSIZE && ro.ro_rt)
-		mtu = ro.ro_rt->rt_rmx.rmx_mtu;
+		mtu = ro.ro_rt->rt_mtu;
 	RO_RTFREE(&ro);
 
 	if (error)

sys/netinet/ip_ipsec.c

@@ -237,9 +237,7 @@ ip_ipsec_mtu(struct mbuf *m, int mtu)
 		    sp->req->sav->sah != NULL) {
 			ro = &sp->req->sav->sah->route_cache.sa_route;
 			if (ro->ro_rt && ro->ro_rt->rt_ifp) {
-				mtu =
-				    ro->ro_rt->rt_rmx.rmx_mtu ?
-				    ro->ro_rt->rt_rmx.rmx_mtu :
+				mtu = ro->ro_rt->rt_mtu ? ro->ro_rt->rt_mtu :
 				    ro->ro_rt->rt_ifp->if_mtu;
 				mtu -= ipsechdr;
 			}

sys/netinet/ip_output.c

@@ -294,7 +294,7 @@ again:
 		}
 		ia = ifatoia(rte->rt_ifa);
 		ifp = rte->rt_ifp;
-		rte->rt_rmx.rmx_pksent++;
+		counter_u64_add(rte->rt_pksent, 1);
 		if (rte->rt_flags & RTF_GATEWAY)
 			gw = (struct sockaddr_in *)rte->rt_gateway;
 		if (rte->rt_flags & RTF_HOST)
@@ -314,9 +314,9 @@ again:
 		 * them, there is no way for one to update all its
 		 * routes when the MTU is changed.
 		 */
-		if (rte->rt_rmx.rmx_mtu > ifp->if_mtu)
-			rte->rt_rmx.rmx_mtu = ifp->if_mtu;
-		mtu = rte->rt_rmx.rmx_mtu;
+		if (rte->rt_mtu > ifp->if_mtu)
+			rte->rt_mtu = ifp->if_mtu;
+		mtu = rte->rt_mtu;
 	} else {
 		mtu = ifp->if_mtu;
 	}

sys/netinet/sctp_os_bsd.h

@@ -328,11 +328,11 @@ typedef struct callout sctp_os_timer_t;
 /*      MTU              */
 /*************************/
 #define SCTP_GATHER_MTU_FROM_IFN_INFO(ifn, ifn_index, af) ((struct ifnet *)ifn)->if_mtu
-#define SCTP_GATHER_MTU_FROM_ROUTE(sctp_ifa, sa, rt) ((rt != NULL) ? rt->rt_rmx.rmx_mtu : 0)
+#define SCTP_GATHER_MTU_FROM_ROUTE(sctp_ifa, sa, rt) ((rt != NULL) ? rt->rt_mtu : 0)
 #define SCTP_GATHER_MTU_FROM_INTFC(sctp_ifn) ((sctp_ifn->ifn_p != NULL) ? ((struct ifnet *)(sctp_ifn->ifn_p))->if_mtu : 0)
 #define SCTP_SET_MTU_OF_ROUTE(sa, rt, mtu) do { \
                                               if (rt != NULL) \
-                                                 rt->rt_rmx.rmx_mtu = mtu; \
+                                                 rt->rt_mtu = mtu; \
                                            } while(0)
 
 /* (de-)register interface event notifications */

sys/netinet/tcp_output.c

@@ -1194,7 +1194,7 @@ send:
 		    NULL, NULL, tp->t_inpcb);
 
 		if (error == EMSGSIZE && ro.ro_rt != NULL)
-			mtu = ro.ro_rt->rt_rmx.rmx_mtu;
+			mtu = ro.ro_rt->rt_mtu;
 		RO_RTFREE(&ro);
 	}
 #endif /* INET6 */
@@ -1232,7 +1232,7 @@ send:
 	    tp->t_inpcb);
 
 	if (error == EMSGSIZE && ro.ro_rt != NULL)
-		mtu = ro.ro_rt->rt_rmx.rmx_mtu;
+		mtu = ro.ro_rt->rt_mtu;
 	RO_RTFREE(&ro);
     }
 #endif /* INET */

sys/netinet/tcp_subr.c

@@ -1811,10 +1811,10 @@ tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
 	}
 	if (sro.ro_rt != NULL) {
 		ifp = sro.ro_rt->rt_ifp;
-		if (sro.ro_rt->rt_rmx.rmx_mtu == 0)
+		if (sro.ro_rt->rt_mtu == 0)
 			maxmtu = ifp->if_mtu;
 		else
-			maxmtu = min(sro.ro_rt->rt_rmx.rmx_mtu, ifp->if_mtu);
+			maxmtu = min(sro.ro_rt->rt_mtu, ifp->if_mtu);
 
 		/* Report additional interface capabilities. */
 		if (cap != NULL) {
@@ -1848,10 +1848,10 @@ tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
 	}
 	if (sro6.ro_rt != NULL) {
 		ifp = sro6.ro_rt->rt_ifp;
-		if (sro6.ro_rt->rt_rmx.rmx_mtu == 0)
+		if (sro6.ro_rt->rt_mtu == 0)
 			maxmtu = IN6_LINKMTU(sro6.ro_rt->rt_ifp);
 		else
-			maxmtu = min(sro6.ro_rt->rt_rmx.rmx_mtu,
+			maxmtu = min(sro6.ro_rt->rt_mtu,
 				     IN6_LINKMTU(sro6.ro_rt->rt_ifp));
 
 		/* Report additional interface capabilities. */

sys/netinet6/in6_rmx.c

@@ -136,8 +136,8 @@ in6_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
 		}
 	}
 
-	if (!rt->rt_rmx.rmx_mtu && rt->rt_ifp)
-		rt->rt_rmx.rmx_mtu = IN6_LINKMTU(rt->rt_ifp);
+	if (!rt->rt_mtu && rt->rt_ifp)
+		rt->rt_mtu = IN6_LINKMTU(rt->rt_ifp);
 
 	ret = rn_addroute(v_arg, n_arg, head, treenodes);
 	if (ret == NULL) {
@@ -207,12 +207,11 @@ in6_mtuexpire(struct radix_node *rn, void *rock)
 	if (!rt)
 		panic("rt == NULL in in6_mtuexpire");
 
-	if (rt->rt_rmx.rmx_expire && !(rt->rt_flags & RTF_PROBEMTU)) {
-		if (rt->rt_rmx.rmx_expire <= time_uptime) {
+	if (rt->rt_expire && !(rt->rt_flags & RTF_PROBEMTU)) {
+		if (rt->rt_expire <= time_uptime) {
 			rt->rt_flags |= RTF_PROBEMTU;
 		} else {
-			ap->nextstop = lmin(ap->nextstop,
-					rt->rt_rmx.rmx_expire);
+			ap->nextstop = lmin(ap->nextstop, rt->rt_expire);
 		}
 	}
 

sys/netinet6/ip6_ipsec.c

@@ -370,9 +370,7 @@ ip6_ipsec_mtu(struct mbuf *m)
 		    sp->req->sav->sah != NULL) {
 			ro = &sp->req->sav->sah->route_cache.sa_route;
 			if (ro->ro_rt && ro->ro_rt->rt_ifp) {
-				mtu =
-				    ro->ro_rt->rt_rmx.rmx_mtu ?
-				    ro->ro_rt->rt_rmx.rmx_mtu :
+				mtu = ro->ro_rt->rt_mtu ? ro->ro_rt->rt_mtu :
 				    ro->ro_rt->rt_ifp->if_mtu;
 				mtu -= ipsechdr;
 			}

sys/netinet6/ip6_output.c

@@ -219,9 +219,9 @@ in6_delayed_cksum(struct mbuf *m, uint32_t plen, u_short offset)
  * skipped and ro->ro_rt would be used. If ro is present but ro->ro_rt is NULL,
  * then result of route lookup is stored in ro->ro_rt.
  *
- * type of "mtu": rt_rmx.rmx_mtu is u_long, ifnet.ifr_mtu is int, and
+ * type of "mtu": rt_mtu is u_long, ifnet.ifr_mtu is int, and
  * nd_ifinfo.linkmtu is u_int32_t.  so we use u_long to hold largest one,
- * which is rt_rmx.rmx_mtu.
+ * which is rt_mtu.
  *
  * ifpp - XXX: just for statistics
  */
@@ -661,7 +661,7 @@ again:
 	}
 	if (rt != NULL) {
 		ia = (struct in6_ifaddr *)(rt->rt_ifa);
-		rt->rt_use++;
+		counter_u64_add(rt->rt_pksent, 1);
 	}
 
 
@@ -1399,9 +1399,9 @@ ip6_getpmtu(struct route_in6 *ro_pmtu, struct route_in6 *ro,
 		ifmtu = IN6_LINKMTU(ifp);
 		mtu = tcp_hc_getmtu(&inc);
 		if (mtu)
-			mtu = min(mtu, ro_pmtu->ro_rt->rt_rmx.rmx_mtu);
+			mtu = min(mtu, ro_pmtu->ro_rt->rt_mtu);
 		else
-			mtu = ro_pmtu->ro_rt->rt_rmx.rmx_mtu;
+			mtu = ro_pmtu->ro_rt->rt_mtu;
 		if (mtu == 0)
 			mtu = ifmtu;
 		else if (mtu < IPV6_MMTU) {
@@ -1425,7 +1425,7 @@ ip6_getpmtu(struct route_in6 *ro_pmtu, struct route_in6 *ro,
 			 * field isn't locked).
 			 */
 			mtu = ifmtu;
-			ro_pmtu->ro_rt->rt_rmx.rmx_mtu = mtu;
+			ro_pmtu->ro_rt->rt_mtu = mtu;
 		}
 	} else if (ifp) {
 		mtu = IN6_LINKMTU(ifp);

sys/netipx/ipx_input.c

@@ -455,7 +455,7 @@ ipx_do_route(struct ipx_addr *src, struct route *ro)
 	if (ro->ro_rt == NULL || ro->ro_rt->rt_ifp == NULL) {
 		return (0);
 	}
-	ro->ro_rt->rt_use++;
+	counter_u64_add(ro->ro_rt->rt_pksent, 1);
 	return (1);
 }
 

sys/netipx/ipx_outputfl.c

@@ -130,7 +130,7 @@ ipx_outputfl(struct mbuf *m0, struct route *ro, int flags)
 		error = ENETUNREACH;
 		goto bad;
 	}
-	ro->ro_rt->rt_use++;
+	counter_u64_add(ro->ro_rt->rt_pksent, 1);
 	if (ro->ro_rt->rt_flags & (RTF_GATEWAY|RTF_HOST))
 		dst = (struct sockaddr_ipx *)ro->ro_rt->rt_gateway;
 gotif:

sys/netpfil/pf/pf.c

@@ -5496,7 +5496,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
 		}
 
 		ifp = rt->rt_ifp;
-		rt->rt_rmx.rmx_pksent++;
+		counter_u64_add(rt->rt_pksent, 1);
 
 		if (rt->rt_flags & RTF_GATEWAY)
 			bcopy(satosin(rt->rt_gateway), &dst, sizeof(dst));

sys/nfs/bootp_subr.c

@@ -343,7 +343,7 @@ bootpboot_p_rtentry(struct rtentry *rt)
 	bootpboot_p_sa(rt->rt_gateway, NULL);
 	printf(" ");
 	printf("flags %x", (unsigned short) rt->rt_flags);
-	printf(" %d", (int) rt->rt_rmx.rmx_expire);
+	printf(" %d", (int) rt->rt_expire);
 	printf(" %s\n", rt->rt_ifp->if_xname);
 }
 

usr.bin/netstat/main.c

@@ -762,6 +762,19 @@ kread(u_long addr, void *buf, size_t size)
 	return (0);
 }
 
+/*
+ * Read single counter(9).
+ */
+uint64_t
+kread_counter(u_long addr)
+{
+
+	if (kvmd_init() < 0)
+		return (-1);
+
+	return (kvm_counter_u64_fetch(kvmd, addr));
+}
+
 /*
  * Read an array of N counters in kernel memory into array of N uint64_t's.
  */

usr.bin/netstat/netstat.h

@@ -60,6 +60,7 @@ extern int	live;	/* true if we are examining a live system */
 
 struct nlist;
 int	kread(u_long addr, void *buf, size_t size);
+uint64_t kread_counter(u_long addr);
 int	kread_counters(u_long addr, void *buf, size_t size);
 int	kresolve_list(struct nlist *);
 const char *plural(uintmax_t);

usr.bin/netstat/route.c

@@ -181,7 +181,7 @@ routepr(int fibnum, int af)
 	/*
 	 * Since kernel & userland use different timebase
 	 * (time_uptime vs time_second) and we are reading kernel memory
-	 * directly we should do rt_rmx.rmx_expire --> expire_time conversion.
+	 * directly we should do rt_expire --> expire_time conversion.
 	 */
 	if (clock_gettime(CLOCK_UPTIME, &uptime) < 0)
 		err(EX_OSERR, "clock_gettime() failed");
@@ -256,8 +256,7 @@ pr_family(int af1)
 static int wid_dst;
 static int wid_gw;
 static int wid_flags;
-static int wid_refs;
-static int wid_use;
+static int wid_pksent;
 static int wid_mtu;
 static int wid_if;
 static int wid_expire;
@@ -268,8 +267,7 @@ size_cols(int ef, struct radix_node *rn)
 	wid_dst = WID_DST_DEFAULT(ef);
 	wid_gw = WID_GW_DEFAULT(ef);
 	wid_flags = 6;
-	wid_refs = 6;
-	wid_use = 8;
+	wid_pksent = 8;
 	wid_mtu = 6;
 	wid_if = WID_IF_DEFAULT(ef);
 	wid_expire = 6;
@@ -329,16 +327,10 @@ size_cols_rtentry(struct rtentry *rt)
 	len = strlen(bp);
 	wid_flags = MAX(len, wid_flags);
 
-	if (addr.u_sa.sa_family == AF_INET || Wflag) {
-		len = snprintf(buffer, sizeof(buffer), "%d", rt->rt_refcnt);
-		wid_refs = MAX(len, wid_refs);
-		len = snprintf(buffer, sizeof(buffer), "%lu", rt->rt_use);
-		wid_use = MAX(len, wid_use);
-		if (Wflag && rt->rt_rmx.rmx_mtu != 0) {
-			len = snprintf(buffer, sizeof(buffer),
-				       "%lu", rt->rt_rmx.rmx_mtu);
-			wid_mtu = MAX(len, wid_mtu);
-		}
+	if (Wflag) {
+		len = snprintf(buffer, sizeof(buffer), "%lu",
+		    kread_counter((u_long )rt->rt_pksent));
+		wid_pksent = MAX(len, wid_pksent);
 	}
 	if (rt->rt_ifp) {
 		if (rt->rt_ifp != lastif) {
@@ -349,11 +341,11 @@ size_cols_rtentry(struct rtentry *rt)
 			lastif = rt->rt_ifp;
 			wid_if = MAX(len, wid_if);
 		}
-		if (rt->rt_rmx.rmx_expire) {
+		if (rt->rt_expire) {
 			time_t expire_time;
 
 			if ((expire_time =
-			    rt->rt_rmx.rmx_expire - uptime.tv_sec) > 0) {
+			    rt->rt_expire - uptime.tv_sec) > 0) {
 				len = snprintf(buffer, sizeof(buffer), "%d",
 					       (int)expire_time);
 				wid_expire = MAX(len, wid_expire);
@@ -373,10 +365,11 @@ pr_rthdr(int af1)
 	if (Aflag)
 		printf("%-8.8s ","Address");
 	if (Wflag) {
-		printf("%-*.*s %-*.*s %-*.*s %*.*s %*.*s %*s\n",
+		printf("%-*.*s %-*.*s %-*.*s %*.*s %*.*s %*.*s %*s\n",
 			wid_dst,	wid_dst,	"Destination",
 			wid_gw,		wid_gw,		"Gateway",
 			wid_flags,	wid_flags,	"Flags",
+			wid_pksent,	wid_pksent,	"Use",
 			wid_mtu,	wid_mtu,	"Mtu",
 			wid_if,		wid_if,		"Netif",
 			wid_expire,			"Expire");
@@ -661,6 +654,8 @@ p_rtentry_sysctl(struct rt_msghdr *rtm)
 	snprintf(buffer, sizeof(buffer), "%%-%d.%ds ", wid_flags, wid_flags);
 	p_flags(rtm->rtm_flags, buffer);
 	if (Wflag) {
+		printf("%*lu ", wid_pksent, rtm->rtm_rmx.rmx_pksent);
+
 		if (rtm->rtm_rmx.rmx_mtu != 0)
 			printf("%*lu ", wid_mtu, rtm->rtm_rmx.rmx_mtu);
 		else
@@ -870,17 +865,14 @@ p_rtentry_kvm(struct rtentry *rt)
 	p_sockaddr(kgetsa(rt->rt_gateway), NULL, RTF_HOST, wid_gw);
 	snprintf(buffer, sizeof(buffer), "%%-%d.%ds ", wid_flags, wid_flags);
 	p_flags(rt->rt_flags, buffer);
-	if (addr.u_sa.sa_family == AF_INET || Wflag) {
-#if 0
-		printf("%*d %*lu ", wid_refs, rt->rt_refcnt,
-				     wid_use, rt->rt_use);
-#endif
-		if (Wflag) {
-			if (rt->rt_rmx.rmx_mtu != 0)
-				printf("%*lu ", wid_mtu, rt->rt_rmx.rmx_mtu);
-			else
-				printf("%*s ", wid_mtu, "");
-		}
+	if (Wflag) {
+		printf("%*lu ", wid_pksent,
+		    kread_counter((u_long )rt->rt_pksent));
+
+		if (rt->rt_mtu != 0)
+			printf("%*lu ", wid_mtu, rt->rt_mtu);
+		else
+			printf("%*s ", wid_mtu, "");
 	}
 	if (rt->rt_ifp) {
 		if (rt->rt_ifp != lastif) {
@@ -892,11 +884,11 @@ p_rtentry_kvm(struct rtentry *rt)
 			lastif = rt->rt_ifp;
 		}
 		printf("%*.*s", wid_if, wid_if, prettyname);
-		if (rt->rt_rmx.rmx_expire) {
+		if (rt->rt_expire) {
 			time_t expire_time;
 
 			if ((expire_time =
-			    rt->rt_rmx.rmx_expire - uptime.tv_sec) > 0)
+			    rt->rt_expire - uptime.tv_sec) > 0)
 				printf(" %*d", wid_expire, (int)expire_time);
 		}
 		if (rt->rt_nodes[0].rn_dupedkey)