cxgbe(4): Stop and restart the L2T allocator with the LLD.

L2T entries are used by both filters and TOE and the L2T is shared
between the base driver (LLD) and the TOM ULD.  Add a flag to indicate
that the L2T is stopped, which means:
* t4_alloc_l2e and t4_l2t_alloc_switching will not allocate new entries.
* t4_tom will ignore all ARP/NDP updates from the kernel.
* Previously allocated L2T entries can still be freed.

MFC after:      1 month
Sponsored by:   Chelsio Communications

sys/dev/cxgbe/t4_l2t.c

@@ -73,7 +73,8 @@ t4_alloc_l2e(struct l2t_data *d)
 	struct l2t_entry *end, *e, **p;
 
 	rw_assert(&d->lock, RA_WLOCKED);
-
+	if (__predict_false(d->l2t_stopped))
+		return (NULL);
 	if (!atomic_load_acq_int(&d->nfree))
 		return (NULL);
 
@@ -291,7 +292,10 @@ t4_l2t_alloc_switching(struct adapter *sc, uint16_t vlan, uint8_t port,
 	int rc;
 
 	rw_wlock(&d->lock);
-	e = find_or_alloc_l2e(d, vlan, port, eth_addr);
+	if (__predict_false(d->l2t_stopped))
+		e = NULL;
+	else
+		e = find_or_alloc_l2e(d, vlan, port, eth_addr);
 	if (e) {
 		if (atomic_load_acq_int(&e->refcnt) == 0) {
 			mtx_lock(&e->lock);    /* avoid race with t4_l2t_free */
@@ -333,6 +337,7 @@ t4_init_l2t(struct adapter *sc, int flags)
 		return (ENOMEM);
 
 	d->l2t_size = l2t_size;
+	d->l2t_stopped = false;
 	d->rover = d->l2tab;
 	atomic_store_rel_int(&d->nfree, l2t_size);
 	rw_init(&d->lock, "L2T");
@@ -353,8 +358,9 @@ t4_init_l2t(struct adapter *sc, int flags)
 }
 
 int
-t4_free_l2t(struct l2t_data *d)
+t4_free_l2t(struct adapter *sc)
 {
+	struct l2t_data *d = sc->l2t;
 	int i;
 
 	for (i = 0; i < d->l2t_size; i++)
@@ -365,6 +371,30 @@ t4_free_l2t(struct l2t_data *d)
 	return (0);
 }
 
+int
+t4_stop_l2t(struct adapter *sc)
+{
+	struct l2t_data *d = sc->l2t;
+
+	rw_wlock(&d->lock);
+	d->l2t_stopped = true;
+	rw_wunlock(&d->lock);
+
+	return (0);
+}
+
+int
+t4_restart_l2t(struct adapter *sc)
+{
+	struct l2t_data *d = sc->l2t;
+
+	rw_wlock(&d->lock);
+	d->l2t_stopped = false;
+	rw_wunlock(&d->lock);
+
+	return (0);
+}
+
 int
 do_l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)

sys/dev/cxgbe/t4_l2t.h

@@ -79,6 +79,7 @@ struct l2t_entry {
 struct l2t_data {
 	struct rwlock lock;
 	u_int l2t_size;
+	bool l2t_stopped;
 	volatile int nfree;	/* number of free entries */
 	struct l2t_entry *rover;/* starting point for next allocation */
 	struct l2t_entry l2tab[];
@@ -86,7 +87,9 @@ struct l2t_data {
 
 
 int t4_init_l2t(struct adapter *, int);
-int t4_free_l2t(struct l2t_data *);
+int t4_free_l2t(struct adapter *);
+int t4_stop_l2t(struct adapter *);
+int t4_restart_l2t(struct adapter *);
 struct l2t_entry *t4_alloc_l2e(struct l2t_data *);
 struct l2t_entry *t4_l2t_alloc_switching(struct adapter *, uint16_t, uint8_t,
     uint8_t *);

sys/dev/cxgbe/t4_main.c

@@ -1847,7 +1847,7 @@ t4_detach_common(device_t dev)
 		    sc->msix_res);
 
 	if (sc->l2t)
-		t4_free_l2t(sc->l2t);
+		t4_free_l2t(sc);
 	if (sc->smt)
 		t4_free_smt(sc->smt);
 	t4_free_atid_table(sc);
@@ -2101,6 +2101,7 @@ stop_lld(struct adapter *sc)
 	end_synchronized_op(sc, 0);
 
 	stop_atid_allocator(sc);
+	t4_stop_l2t(sc);
 
 	return (rc);
 }
@@ -2454,6 +2455,7 @@ done:
 	free(old_state, M_CXGBE);
 
 	restart_atid_allocator(sc);
+	t4_restart_l2t(sc);
 
 	return (rc);
 }

sys/dev/cxgbe/tom/t4_tom_l2t.c

@@ -380,6 +380,10 @@ t4_l2t_get(struct port_info *pi, if_t ifp, struct sockaddr *sa)
 
 	hash = l2_hash(d, sa, if_getindex(ifp));
 	rw_wlock(&d->lock);
+	if (__predict_false(d->l2t_stopped)) {
+		e = NULL;
+		goto done;
+	}
 	for (e = d->l2tab[hash].first; e; e = e->next) {
 		if (l2_cmp(sa, e) == 0 && e->ifp == ifp && e->vlan == vtag &&
 		    e->smt_idx == smt_idx) {
@@ -429,6 +433,8 @@ t4_l2_update(struct toedev *tod, if_t ifp, struct sockaddr *sa,
 
 	hash = l2_hash(d, sa, if_getindex(ifp));
 	rw_rlock(&d->lock);
+	if (__predict_false(d->l2t_stopped))
+		goto done;
 	for (e = d->l2tab[hash].first; e; e = e->next) {
 		if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
 			mtx_lock(&e->lock);
@@ -439,6 +445,7 @@ t4_l2_update(struct toedev *tod, if_t ifp, struct sockaddr *sa,
 			break;
 		}
 	}
+done:
 	rw_runlock(&d->lock);
 
 	/*