Add an IOCTL rr_limit to let users fine tuning the number of packets to be

sent using roundrobin protocol and set a better granularity and distribution
among the interfaces. Tuning the number of packages sent by interface can
increase throughput and reduce unordered packets as well as reduce SACK.

Example of usage:
# ifconfig bge0 up
# ifconfig bge1 up
# ifconfig lagg0 create
# ifconfig lagg0 laggproto roundrobin laggport bge0 laggport bge1 \
	192.168.1.1 netmask 255.255.255.0
# ifconfig lagg0 rr_limit 500

Reviewed by:	thompsa, glebius, adrian (old patch)
Approved by:	bapt (mentor)
Relnotes:	Yes
Differential Revision:	https://reviews.freebsd.org/D540

sbin/ifconfig/iflagg.c

@@ -99,6 +99,19 @@ setlaggflowidshift(const char *val, int d, int s, const struct afswtch *afp)
 		err(1, "SIOCSLAGGOPTS");
 }
 
+static void
+setlaggrr_limit(const char *val, int d, int s, const struct afswtch *afp)
+{
+	struct lagg_reqopts ro;
+	
+	bzero(&ro, sizeof(ro));
+	strlcpy(ro.ro_ifname, name, sizeof(ro.ro_ifname));
+	ro.ro_bkt = (int)strtol(val, NULL, 10);
+
+	if (ioctl(s, SIOCSLAGGOPTS, &ro) != 0)
+		err(1, "SIOCSLAGG");
+}
+
 static void
 setlaggsetopt(const char *val, int d, int s, const struct afswtch *afp)
 {
@@ -252,6 +265,8 @@ lagg_status(int s)
 			printb("\t\tflags", ro.ro_opts, LAGG_OPT_BITS);
 			putchar('\n');
 			printf("\t\tflowid_shift: %d\n", ro.ro_flowid_shift);
+			if (ra.ra_proto == LAGG_PROTO_ROUNDROBIN)
+				printf("\t\trr_limit: %d\n", ro.ro_bkt);
 			printf("\tlagg statistics:\n");
 			printf("\t\tactive ports: %d\n", ro.ro_active);
 			printf("\t\tflapping: %u\n", ro.ro_flapping);
@@ -298,6 +313,7 @@ static struct cmd lagg_cmds[] = {
 	DEF_CMD("lacp_fast_timeout",	LAGG_OPT_LACP_TIMEOUT,	setlaggsetopt),
 	DEF_CMD("-lacp_fast_timeout",	-LAGG_OPT_LACP_TIMEOUT,	setlaggsetopt),
 	DEF_CMD_ARG("flowid_shift",	setlaggflowidshift),
+	DEF_CMD_ARG("rr_limit",		setlaggrr_limit),
 };
 static struct afswtch af_lagg = {
 	.af_name	= "af_lagg",

share/man/man4/lagg.4

@@ -16,7 +16,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 6, 2015
+.Dd January 23, 2016
 .Dt LAGG 4
 .Os
 .Sh NAME
@@ -110,6 +110,11 @@ available, the VLAN tag, and the IP source and destination address.
 Distributes outgoing traffic using a round-robin scheduler
 through all active ports and accepts incoming traffic from
 any active port.
+Using
+.Ic roundrobin
+mode can cause unordered packet arrival at the client.
+Throughput might be limited as the client performs CPU-intensive packet
+reordering.
 .It Ic broadcast
 Sends frames to all ports of the LAG and receives frames on
 any port of the LAG.
@@ -161,6 +166,19 @@ Gigabit Ethernet interfaces:
 	192.168.1.1 netmask 255.255.255.0
 .Ed
 .Pp
+Create a link aggregation using ROUNDROBIN with two 
+.Xr bge 4
+Gigabit Ethernet interfaces and set the limit of 500 packets
+per interface:
+.Bd -literal -offset indent
+# ifconfig bge0 up
+# ifconfig bge1 up
+# ifconfig lagg0 create
+# ifconfig lagg0 laggproto roundrobin laggport bge0 laggport bge1 \e
+	192.168.1.1 netmask 255.255.255.0
+# ifconfig lagg0 rr_limit 500
+.Ed
+.Pp
 The following example uses an active failover interface to set up roaming
 between wired and wireless networks using two network devices.
 Whenever the wired master interface is unplugged, the wireless failover

sys/net/if_lagg.c

@@ -3,7 +3,7 @@
 /*
  * Copyright (c) 2005, 2006 Reyk Floeter <reyk@openbsd.org>
  * Copyright (c) 2007 Andrew Thompson <thompsa@FreeBSD.org>
- * Copyright (c) 2014 Marcelo Araujo <araujo@FreeBSD.org>
+ * Copyright (c) 2014, 2016 Marcelo Araujo <araujo@FreeBSD.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -1291,10 +1291,17 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			SLIST_FOREACH(lp, &sc->sc_ports, lp_entries)
 				ro->ro_active += LAGG_PORTACTIVE(lp);
 		}
+		ro->ro_bkt = sc->sc_bkt;
 		ro->ro_flapping = sc->sc_flapping;
 		ro->ro_flowid_shift = sc->flowid_shift;
 		break;
 	case SIOCSLAGGOPTS:
+		if (sc->sc_proto == LAGG_PROTO_ROUNDROBIN) {
+			if (ro->ro_bkt == 0)
+				sc->sc_bkt = 1; // Minimum 1 packet per iface.
+			else
+				sc->sc_bkt = ro->ro_bkt;
+		}
 		error = priv_check(td, PRIV_NET_LAGG);
 		if (error)
 			break;
@@ -1329,6 +1336,7 @@ lagg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		}
 
 		LAGG_WLOCK(sc);
+
 		if (valid == 0 ||
 		    (lacp == 1 && sc->sc_proto != LAGG_PROTO_LACP)) {
 			/* Invalid combination of options specified. */
@@ -1879,6 +1887,7 @@ lagg_rr_attach(struct lagg_softc *sc)
 {
 	sc->sc_capabilities = IFCAP_LAGG_FULLDUPLEX;
 	sc->sc_seq = 0;
+	sc->sc_bkt_count = sc->sc_bkt;
 }
 
 static int
@@ -1887,9 +1896,21 @@ lagg_rr_start(struct lagg_softc *sc, struct mbuf *m)
 	struct lagg_port *lp;
 	uint32_t p;
 
-	p = atomic_fetchadd_32(&sc->sc_seq, 1);
+	if (sc->sc_bkt_count == 0 && sc->sc_bkt > 0)
+		sc->sc_bkt_count = sc->sc_bkt;
+
+	if (sc->sc_bkt > 0) {
+		atomic_subtract_int(&sc->sc_bkt_count, 1);
+	if (atomic_cmpset_int(&sc->sc_bkt_count, 0, sc->sc_bkt))
+		p = atomic_fetchadd_32(&sc->sc_seq, 1);
+	else
+		p = sc->sc_seq; 
+	} else
+		p = atomic_fetchadd_32(&sc->sc_seq, 1);
+
 	p %= sc->sc_count;
 	lp = SLIST_FIRST(&sc->sc_ports);
+
 	while (p--)
 		lp = SLIST_NEXT(lp, lp_entries);
 

sys/net/if_lagg.h

@@ -153,6 +153,7 @@ struct lagg_reqopts {
 	u_int			ro_active;		/* active port count */
 	u_int			ro_flapping;		/* number of flapping */
 	int			ro_flowid_shift;	/* shift the flowid */
+	uint32_t		ro_bkt;			/* packet bucket for roundrobin */
 };
 
 #define	SIOCGLAGGOPTS		_IOWR('i', 152, struct lagg_reqopts)
@@ -243,6 +244,8 @@ struct lagg_softc {
 	struct callout			sc_callout;
 	u_int				sc_opts;
 	int				flowid_shift;	/* shift the flowid */
+	uint32_t			sc_bkt;		/* packates bucket for roundrobin */
+	uint32_t			sc_bkt_count;	/* packates bucket count for roundrobin */
 	struct lagg_counters		detached_counters; /* detached ports sum */
 };