From d62edc5eb5361eb89be1f94a30cf64153575a483 Mon Sep 17 00:00:00 2001 From: Marcelo Araujo Date: Sat, 23 Jan 2016 04:18:44 +0000 Subject: [PATCH] Add an IOCTL rr_limit to let users fine tuning the number of packets to be sent using roundrobin protocol and set a better granularity and distribution among the interfaces. Tuning the number of packages sent by interface can increase throughput and reduce unordered packets as well as reduce SACK. Example of usage: # ifconfig bge0 up # ifconfig bge1 up # ifconfig lagg0 create # ifconfig lagg0 laggproto roundrobin laggport bge0 laggport bge1 \ 192.168.1.1 netmask 255.255.255.0 # ifconfig lagg0 rr_limit 500 Reviewed by: thompsa, glebius, adrian (old patch) Approved by: bapt (mentor) Relnotes: Yes Differential Revision: https://reviews.freebsd.org/D540 --- sbin/ifconfig/iflagg.c | 16 ++++++++++++++++ share/man/man4/lagg.4 | 20 +++++++++++++++++++- sys/net/if_lagg.c | 25 +++++++++++++++++++++++-- sys/net/if_lagg.h | 3 +++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/sbin/ifconfig/iflagg.c b/sbin/ifconfig/iflagg.c index 97b42e31e431..cbf58f7ba908 100644 --- a/sbin/ifconfig/iflagg.c +++ b/sbin/ifconfig/iflagg.c @@ -99,6 +99,19 @@ setlaggflowidshift(const char *val, int d, int s, const struct afswtch *afp) err(1, "SIOCSLAGGOPTS"); } +static void +setlaggrr_limit(const char *val, int d, int s, const struct afswtch *afp) +{ + struct lagg_reqopts ro; + + bzero(&ro, sizeof(ro)); + strlcpy(ro.ro_ifname, name, sizeof(ro.ro_ifname)); + ro.ro_bkt = (int)strtol(val, NULL, 10); + + if (ioctl(s, SIOCSLAGGOPTS, &ro) != 0) + err(1, "SIOCSLAGG"); +} + static void setlaggsetopt(const char *val, int d, int s, const struct afswtch *afp) { @@ -252,6 +265,8 @@ lagg_status(int s) printb("\t\tflags", ro.ro_opts, LAGG_OPT_BITS); putchar('\n'); printf("\t\tflowid_shift: %d\n", ro.ro_flowid_shift); + if (ra.ra_proto == LAGG_PROTO_ROUNDROBIN) + printf("\t\trr_limit: %d\n", ro.ro_bkt); printf("\tlagg statistics:\n"); printf("\t\tactive ports: %d\n", ro.ro_active); printf("\t\tflapping: %u\n", ro.ro_flapping); @@ -298,6 +313,7 @@ static struct cmd lagg_cmds[] = { DEF_CMD("lacp_fast_timeout", LAGG_OPT_LACP_TIMEOUT, setlaggsetopt), DEF_CMD("-lacp_fast_timeout", -LAGG_OPT_LACP_TIMEOUT, setlaggsetopt), DEF_CMD_ARG("flowid_shift", setlaggflowidshift), + DEF_CMD_ARG("rr_limit", setlaggrr_limit), }; static struct afswtch af_lagg = { .af_name = "af_lagg", diff --git a/share/man/man4/lagg.4 b/share/man/man4/lagg.4 index da070f158d0e..c1a1bef4d926 100644 --- a/share/man/man4/lagg.4 +++ b/share/man/man4/lagg.4 @@ -16,7 +16,7 @@ .\" .\" $FreeBSD$ .\" -.Dd November 6, 2015 +.Dd January 23, 2016 .Dt LAGG 4 .Os .Sh NAME @@ -110,6 +110,11 @@ available, the VLAN tag, and the IP source and destination address. Distributes outgoing traffic using a round-robin scheduler through all active ports and accepts incoming traffic from any active port. +Using +.Ic roundrobin +mode can cause unordered packet arrival at the client. +Throughput might be limited as the client performs CPU-intensive packet +reordering. .It Ic broadcast Sends frames to all ports of the LAG and receives frames on any port of the LAG. @@ -161,6 +166,19 @@ Gigabit Ethernet interfaces: 192.168.1.1 netmask 255.255.255.0 .Ed .Pp +Create a link aggregation using ROUNDROBIN with two +.Xr bge 4 +Gigabit Ethernet interfaces and set the limit of 500 packets +per interface: +.Bd -literal -offset indent +# ifconfig bge0 up +# ifconfig bge1 up +# ifconfig lagg0 create +# ifconfig lagg0 laggproto roundrobin laggport bge0 laggport bge1 \e + 192.168.1.1 netmask 255.255.255.0 +# ifconfig lagg0 rr_limit 500 +.Ed +.Pp The following example uses an active failover interface to set up roaming between wired and wireless networks using two network devices. Whenever the wired master interface is unplugged, the wireless failover diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c index 730a044a8f12..e1e1837f3d7c 100644 --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -3,7 +3,7 @@ /* * Copyright (c) 2005, 2006 Reyk Floeter * Copyright (c) 2007 Andrew Thompson - * Copyright (c) 2014 Marcelo Araujo + * Copyright (c) 2014, 2016 Marcelo Araujo * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -1291,10 +1291,17 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) SLIST_FOREACH(lp, &sc->sc_ports, lp_entries) ro->ro_active += LAGG_PORTACTIVE(lp); } + ro->ro_bkt = sc->sc_bkt; ro->ro_flapping = sc->sc_flapping; ro->ro_flowid_shift = sc->flowid_shift; break; case SIOCSLAGGOPTS: + if (sc->sc_proto == LAGG_PROTO_ROUNDROBIN) { + if (ro->ro_bkt == 0) + sc->sc_bkt = 1; // Minimum 1 packet per iface. + else + sc->sc_bkt = ro->ro_bkt; + } error = priv_check(td, PRIV_NET_LAGG); if (error) break; @@ -1329,6 +1336,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) } LAGG_WLOCK(sc); + if (valid == 0 || (lacp == 1 && sc->sc_proto != LAGG_PROTO_LACP)) { /* Invalid combination of options specified. */ @@ -1879,6 +1887,7 @@ lagg_rr_attach(struct lagg_softc *sc) { sc->sc_capabilities = IFCAP_LAGG_FULLDUPLEX; sc->sc_seq = 0; + sc->sc_bkt_count = sc->sc_bkt; } static int @@ -1887,9 +1896,21 @@ lagg_rr_start(struct lagg_softc *sc, struct mbuf *m) struct lagg_port *lp; uint32_t p; - p = atomic_fetchadd_32(&sc->sc_seq, 1); + if (sc->sc_bkt_count == 0 && sc->sc_bkt > 0) + sc->sc_bkt_count = sc->sc_bkt; + + if (sc->sc_bkt > 0) { + atomic_subtract_int(&sc->sc_bkt_count, 1); + if (atomic_cmpset_int(&sc->sc_bkt_count, 0, sc->sc_bkt)) + p = atomic_fetchadd_32(&sc->sc_seq, 1); + else + p = sc->sc_seq; + } else + p = atomic_fetchadd_32(&sc->sc_seq, 1); + p %= sc->sc_count; lp = SLIST_FIRST(&sc->sc_ports); + while (p--) lp = SLIST_NEXT(lp, lp_entries); diff --git a/sys/net/if_lagg.h b/sys/net/if_lagg.h index 195ac3a677af..334995e5c337 100644 --- a/sys/net/if_lagg.h +++ b/sys/net/if_lagg.h @@ -153,6 +153,7 @@ struct lagg_reqopts { u_int ro_active; /* active port count */ u_int ro_flapping; /* number of flapping */ int ro_flowid_shift; /* shift the flowid */ + uint32_t ro_bkt; /* packet bucket for roundrobin */ }; #define SIOCGLAGGOPTS _IOWR('i', 152, struct lagg_reqopts) @@ -243,6 +244,8 @@ struct lagg_softc { struct callout sc_callout; u_int sc_opts; int flowid_shift; /* shift the flowid */ + uint32_t sc_bkt; /* packates bucket for roundrobin */ + uint32_t sc_bkt_count; /* packates bucket count for roundrobin */ struct lagg_counters detached_counters; /* detached ports sum */ };