gconcat: Implement new online append feature

Implement the "gconcat append" command which can be used
to append a disk to the end of an existing gconcat device
without unmounting.

If the gconcat device is using the "automatic" method, i.e.,
stores metadata on the devices, new metadata is written
to all existing components, as well as to the newly added one.

Pull Request:	https://github.com/freebsd/freebsd-src/pull/472
Reviewed by:	imp@

lib/geom/concat/gconcat.8

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 21, 2004
+.Dd June 14, 2021
 .Dt GCONCAT 8
 .Os
 .Sh NAME
@@ -46,6 +46,11 @@
 .Ar name
 .Ar prov ...
 .Nm
+.Cm append
+.Op Fl hv
+.Ar name
+.Ar prov
+.Nm
 .Cm stop
 .Op Fl fv
 .Ar name ...
@@ -104,10 +109,44 @@ method, where metadata are stored in every device's last sector.
 The kernel module
 .Pa geom_concat.ko
 will be loaded if it is not loaded already.
+.Pp
+Additional options include:
+.Bl -tag -width ".Fl h"
+.It Fl h
+Hardcode providers' names in metadata.
+.El
+.It Cm append
+Append a new device to the end of an existing concatenate device
+with the specified
+.Ar name .
+.Pp
+If the existing device is using the
+.Dq manual
+method, the new device is simply appended as-is.
+.Pp
+If the existing device is using the
+.Dq automatic
+method, the device is appended persistently.
+New
+.Cm gconcat
+metadata is written to all existing components, as well as to the
+newly added one.
+.Pp
+Additional options include:
+.Bl -tag -width ".Fl h"
+.It Fl h
+Hardcode providers' names in metadata.
+.El
 .It Cm stop
 Turn off existing concatenate device by its
 .Ar name .
 This command does not touch on-disk metadata!
+.Pp
+Additional options include:
+.Bl -tag -width ".Fl f"
+.It Fl f
+Stop the given device even if it is opened.
+.El
 .It Cm destroy
 Same as
 .Cm stop .
@@ -131,10 +170,6 @@ See
 .Pp
 Additional options:
 .Bl -tag -width indent
-.It Fl f
-Force the removal of the specified concatenated device.
-.It Fl h
-Hardcode providers' names in metadata.
 .It Fl v
 Be more verbose.
 .El

lib/geom/concat/geom_concat.c

@@ -53,6 +53,13 @@ static void concat_dump(struct gctl_req *req);
 static void concat_label(struct gctl_req *req);
 
 struct g_command class_commands[] = {
+	{ "append", G_FLAG_VERBOSE, NULL,
+	    {
+		{ 'h', "hardcode", NULL, G_TYPE_BOOL },
+		G_OPT_SENTINEL
+	    },
+	    "[-hv] name prov"
+	},
 	{ "clear", G_FLAG_VERBOSE, concat_main, G_NULL_OPTS,
 	    "[-v] prov ..."
 	},

sys/geom/concat/g_concat.c

@@ -595,6 +595,10 @@ g_concat_add_disk(struct g_concat_softc *sc, struct g_provider *pp, u_int no)
 			G_CONCAT_DEBUG(0, "Metadata on %s changed.", pp->name);
 			goto fail;
 		}
+
+		disk->d_hardcoded = md.md_provider[0] != '\0';
+	} else {
+		disk->d_hardcoded = false;
 	}
 
 	cp->private = disk;
@@ -984,6 +988,203 @@ g_concat_ctl_destroy(struct gctl_req *req, struct g_class *mp)
 	}
 }
 
+static struct g_concat_disk *
+g_concat_find_disk(struct g_concat_softc *sc, const char *name)
+{
+	struct g_concat_disk *disk;
+
+	sx_assert(&sc->sc_disks_lock, SX_LOCKED);
+	if (strncmp(name, "/dev/", 5) == 0)
+		name += 5;
+	TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
+		if (disk->d_consumer == NULL)
+			continue;
+		if (disk->d_consumer->provider == NULL)
+			continue;
+		if (strcmp(disk->d_consumer->provider->name, name) == 0)
+			return (disk);
+	}
+	return (NULL);
+}
+
+static void
+g_concat_write_metadata(struct gctl_req *req, struct g_concat_softc *sc)
+{
+	u_int no = 0;
+	struct g_concat_disk *disk;
+	struct g_concat_metadata md;
+	struct g_provider *pp;
+	u_char *sector;
+	int error;
+
+	strlcpy(md.md_magic, G_CONCAT_MAGIC, sizeof(md.md_magic));
+	md.md_version = G_CONCAT_VERSION;
+	strlcpy(md.md_name, sc->sc_name, sizeof(md.md_name));
+	md.md_id = sc->sc_id;
+	md.md_all = sc->sc_ndisks;
+	TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
+		pp = disk->d_consumer->provider;
+
+		md.md_no = no;
+		bzero(md.md_provider, sizeof(md.md_provider));
+		if (disk->d_hardcoded) {
+			strlcpy(md.md_provider, pp->name, sizeof(md.md_provider));
+		}
+		md.md_provsize = disk->d_consumer->provider->mediasize;
+
+		sector = g_malloc(pp->sectorsize, M_WAITOK);
+
+		concat_metadata_encode(&md, sector);
+		error = g_access(disk->d_consumer, 0, 1, 0);
+		if (error == 0) {
+			error = g_write_data(disk->d_consumer, pp->mediasize - pp->sectorsize,
+			    sector, pp->sectorsize);
+			(void)g_access(disk->d_consumer, 0, -1, 0);
+		}
+		g_free(sector);
+		if (error != 0) {
+			gctl_error(req, "Cannot store metadata on %s: %d", pp->name, error);
+		}
+
+		no++;
+	}
+}
+
+static void
+g_concat_ctl_append(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_concat_softc *sc;
+	struct g_consumer *cp, *fcp;
+	struct g_provider *pp;
+	struct g_geom *gp;
+	const char *name, *cname;
+	struct g_concat_disk *disk;
+	int *nargs, *hardcode;
+	int error;
+	int disk_candelete;
+
+	g_topology_assert();
+
+	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
+	if (nargs == NULL) {
+		gctl_error(req, "No '%s' argument.", "nargs");
+		return;
+	}
+	if (*nargs != 2) {
+		gctl_error(req, "Invalid number of arguments.");
+		return;
+	}
+	hardcode = gctl_get_paraml(req, "hardcode", sizeof(*hardcode));
+	if (hardcode == NULL) {
+		gctl_error(req, "No '%s' argument.", "hardcode");
+		return;
+	}
+
+	cname = gctl_get_asciiparam(req, "arg0");
+	if (cname == NULL) {
+		gctl_error(req, "No 'arg%u' argument.", 0);
+		return;
+	}
+	sc = g_concat_find_device(mp, cname);
+	if (sc == NULL) {
+		gctl_error(req, "No such device: %s.", cname);
+		return;
+	}
+	if (sc->sc_provider == NULL) {
+		/*
+		 * this won't race with g_concat_remove_disk as both
+		 * are holding the topology lock
+		 */
+		gctl_error(req, "Device not active, can't append: %s.", cname);
+		return;
+	}
+	G_CONCAT_DEBUG(1, "Appending to %s:", cname);
+	sx_xlock(&sc->sc_disks_lock);
+	gp = sc->sc_geom;
+	fcp = LIST_FIRST(&gp->consumer);
+
+	name = gctl_get_asciiparam(req, "arg1");
+	if (name == NULL) {
+		gctl_error(req, "No 'arg%u' argument.", 1);
+		goto fail;
+	}
+	if (strncmp(name, "/dev/", strlen("/dev/")) == 0)
+		name += strlen("/dev/");
+	pp = g_provider_by_name(name);
+	if (pp == NULL) {
+		G_CONCAT_DEBUG(1, "Disk %s is invalid.", name);
+		gctl_error(req, "Disk %s is invalid.", name);
+		goto fail;
+	}
+	G_CONCAT_DEBUG(1, "Appending %s to this", name);
+
+	if (g_concat_find_disk(sc, name) != NULL) {
+		gctl_error(req, "Disk %s already appended.", name);
+		goto fail;
+	}
+
+	if ((sc->sc_provider->sectorsize % pp->sectorsize) != 0) {
+		gctl_error(req, "Providers sectorsize mismatch: %u vs %u",
+			   sc->sc_provider->sectorsize, pp->sectorsize);
+		goto fail;
+	}
+
+	cp = g_new_consumer(gp);
+	cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
+	error = g_attach(cp, pp);
+	if (error != 0) {
+		g_destroy_consumer(cp);
+		gctl_error(req, "Cannot open device %s (error=%d).",
+		    name, error);
+		goto fail;
+	}
+
+	error = g_access(cp, 1, 0, 0);
+	if (error == 0) {
+		error = g_getattr("GEOM::candelete", cp, &disk_candelete);
+		if (error != 0)
+			disk_candelete = 0;
+		(void)g_access(cp, -1, 0, 0);
+	} else
+		G_CONCAT_DEBUG(1, "Failed to access disk %s, error %d.", name, error);
+
+	/* invoke g_access exactly as deep as all the other members currently are */
+	if (fcp != NULL && (fcp->acr > 0 || fcp->acw > 0 || fcp->ace > 0)) {
+		error = g_access(cp, fcp->acr, fcp->acw, fcp->ace);
+		if (error != 0) {
+			g_detach(cp);
+			g_destroy_consumer(cp);
+			gctl_error(req, "Failed to access disk %s (error=%d).", name, error);
+			goto fail;
+		}
+	}
+
+	disk = malloc(sizeof(*disk), M_CONCAT, M_WAITOK | M_ZERO);
+	disk->d_consumer = cp;
+	disk->d_softc = sc;
+	disk->d_start = TAILQ_LAST(&sc->sc_disks, g_concat_disks)->d_end;
+	disk->d_end = disk->d_start + cp->provider->mediasize;
+	disk->d_candelete = disk_candelete;
+	disk->d_removed = 0;
+	disk->d_hardcoded = *hardcode;
+	cp->private = disk;
+	TAILQ_INSERT_TAIL(&sc->sc_disks, disk, d_next);
+	sc->sc_ndisks++;
+
+	if (sc->sc_type == G_CONCAT_TYPE_AUTOMATIC) {
+		/* last sector is for metadata */
+		disk->d_end -= cp->provider->sectorsize;
+
+		/* update metadata on all parts */
+		g_concat_write_metadata(req, sc);
+	}
+
+	g_resize_provider(sc->sc_provider, disk->d_end);
+
+fail:
+	sx_xunlock(&sc->sc_disks_lock);
+}
+
 static void
 g_concat_config(struct gctl_req *req, struct g_class *mp, const char *verb)
 {
@@ -1008,6 +1209,9 @@ g_concat_config(struct gctl_req *req, struct g_class *mp, const char *verb)
 	    strcmp(verb, "stop") == 0) {
 		g_concat_ctl_destroy(req, mp);
 		return;
+	} else if (strcmp(verb, "append") == 0) {
+		g_concat_ctl_append(req, mp);
+		return;
 	}
 	gctl_error(req, "Unknown verb.");
 }

sys/geom/concat/g_concat.h

@@ -62,6 +62,7 @@ struct g_concat_disk {
 	off_t			 d_end;
 	int			 d_candelete;
 	int			 d_removed;
+	bool			 d_hardcoded;
 };
 
 struct g_concat_softc {