After r290196, the kernel won't wait for stuff like gmirror nodes

if they are not required for mounting rootfs.  However, it's possible
that some setups try to mount them in mountcritlocal (ie from fstab).

Export the list of current root mount holds using a new sysctl,
vfs.root_mount_hold, and make mountcritlocal retry if "mount -a" fails
and the list is not empty.

MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3709

etc/defaults/rc.conf

@@ -84,6 +84,7 @@ geli_autodetach="YES"	# Automatically detach on last close.
 #geli_mirror_home_flags="-k /etc/geli/home.keys"
 
 root_rw_mount="YES"	# Set to NO to inhibit remounting root read-write.
+root_hold_delay="30"	# Time to wait for root mount hold release.
 fsck_y_enable="NO"	# Set to YES to do fsck -y if the initial preen fails.
 fsck_y_flags=""		# Additional flags for fsck -y
 background_fsck="YES"	# Attempt to run fsck in the background where possible.

etc/rc.d/mountcritlocal

@@ -15,7 +15,7 @@ stop_cmd=sync
 
 mountcritlocal_start()
 {
-	local err
+	local err holders waited
 
 	# Set up the list of network filesystem types for which mounting
 	# should be delayed until after network initialization.
@@ -35,8 +35,42 @@ mountcritlocal_start()
 		mount_excludes="${mount_excludes}${fstype},"
 	done
 	mount_excludes=${mount_excludes%,}
+
+	# Originally, root mount hold had to be released before mounting the root
+	# filesystem.  This delayed the boot, so it was changed to only wait if
+	# the root device isn't readily available.  This can result in this script
+	# executing before all the devices - such as graid(8) - are available.
+	# Thus, should the mount fail, we will wait for the root mount hold release
+	# and retry.
 	mount -a -t ${mount_excludes}
 	err=$?
+	if [ $? -ne 0 ]; then
+		echo
+		echo 'Mounting /etc/fstab filesystems failed,' \
+		    'will retry after root mount hold release'
+
+		waited=0
+		while [ ${waited} -lt ${root_hold_delay} ]; do
+			holders="$(sysctl -n vfs.root_mount_hold)"
+			if [ -z "${holders}" ]; then
+				break;
+			fi
+			if [ ${waited} -eq 0 ]; then
+				echo -n "Waiting ${root_hold_delay}s" \
+				"for the root mount holders: ${holders}"
+			else
+				echo -n .
+			fi
+			if [ ${waited} -eq ${root_hold_delay} ]; then
+				break 2
+			fi
+			sleep 1
+			waited=$(($waited + 1))
+		done
+		mount -a -t ${mount_excludes}
+		err=$?
+	fi
+
 	check_startmsgs && echo '.'
 
 	case ${err} in
@@ -44,7 +78,7 @@ mountcritlocal_start()
 		;;
 	*)
 		echo 'Mounting /etc/fstab filesystems failed,' \
-		    ' startup aborted'
+		    'startup aborted'
 		stop_boot true
 		;;
 	esac

sys/kern/vfs_mountroot.c

@@ -88,6 +88,7 @@ __FBSDID("$FreeBSD$");
 
 static int parse_mount(char **);
 static struct mntarg *parse_mountroot_options(struct mntarg *, const char *);
+static int sysctl_vfs_root_mount_hold(SYSCTL_HANDLER_ARGS);
 static int vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev);
 
 /*
@@ -130,6 +131,35 @@ static int root_mount_complete;
 static int root_mount_timeout = 3;
 TUNABLE_INT("vfs.mountroot.timeout", &root_mount_timeout);
 
+SYSCTL_PROC(_vfs, OID_AUTO, root_mount_hold,
+    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
+    NULL, 0, sysctl_vfs_root_mount_hold, "A",
+    "List of root mount hold tokens");
+
+static int
+sysctl_vfs_root_mount_hold(SYSCTL_HANDLER_ARGS)
+{
+	struct sbuf sb;
+	struct root_hold_token *h;
+	int error;
+
+	sbuf_new(&sb, NULL, 256, SBUF_AUTOEXTEND | SBUF_INCLUDENUL);
+
+	mtx_lock(&root_holds_mtx);
+	LIST_FOREACH(h, &root_holds, list) {
+		if (h != LIST_FIRST(&root_holds))
+			sbuf_putc(&sb, ' ');
+		sbuf_printf(&sb, "%s", h->who);
+	}
+	mtx_unlock(&root_holds_mtx);
+
+	error = sbuf_finish(&sb);
+	if (error == 0)
+		error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
+	sbuf_delete(&sb);
+	return (error);
+}
+
 struct root_hold_token *
 root_mount_hold(const char *identifier)
 {