Rename the GLOB_MAXPATH flag of glob(3) to GLOB_LIMIT to be compatible

with NetBSD and OpenBSD. glob(3) will now return GLOB_NOSPACE with
errno set to 0 instead of GLOB_LIMIT when we match more than `gl_matchc'
patterns. GLOB_MAXPATH has been left as an alias of GLOB_LIMIT to
maintain backwards compatibility.

Reviewed by:	sheldonh, assar
Obtained from:	NetBSD/OpenBSD

include/glob.h

@@ -77,11 +77,13 @@ typedef struct {
 #define	GLOB_NOMAGIC	0x0200	/* GLOB_NOCHECK without magic chars (csh). */
 #define	GLOB_QUOTE	0x0400	/* Quote special chars with \. */
 #define	GLOB_TILDE	0x0800	/* Expand tilde names from the passwd file. */
-#define	GLOB_MAXPATH	0x1000	/* limit number of returned paths */
+#define	GLOB_LIMIT	0x1000	/* limit number of returned paths */
+
+/* backwards compatibility, this is the old name for this option */
+#define GLOB_MAXPATH	GLOB_LIMIT
 
 #define	GLOB_NOSPACE	(-1)	/* Malloc call failed. */
 #define	GLOB_ABEND	(-2)	/* Unignored error. */
-#define	GLOB_LIMIT	(-3)	/* Path limit was hit. */
 
 __BEGIN_DECLS
 int	glob __P((const char *, int, int (*)(const char *, int), glob_t *));

lib/libc/gen/glob.3

@@ -260,14 +260,15 @@ character, avoiding any special interpretation of the character.
 Expand patterns that start with
 .Ql ~
 to user name home directories.
-.It Dv GLOB_MAXPATH
+.It Dv GLOB_LIMIT
 Limit the total number of returned pathnames to the value provided in
-.Fa gl_matchc .
-If
-.Fn glob
-would match more pathnames,
-.Dv GLOB_LIMIT
-will be returned.
+.Fa gl_matchc
+(default ARG_MAX).
+This option should be set for programs
+that can be coerced into a denial of service attack
+via patterns that expand to a very large number of matches,
+such as a long string of
+.Ql */../*/.. .
 .El
 .Pp
 If, during the search, a directory is encountered that cannot be opened
@@ -377,21 +378,19 @@ file
 .Aq Pa glob.h :
 .Bl -tag -width GLOB_NOCHECK
 .It Dv GLOB_NOSPACE
-An attempt to allocate memory failed.
+An attempt to allocate memory failed, or if
+.Fa errno
+was 0
+.Dv GLOB_LIMIT
+was specified in the flags and
+.Fa pglob\->gl_matchc
+or more patterns were patched.
 .It Dv GLOB_ABEND
 The scan was stopped because an error was encountered and either
 .Dv GLOB_ERR
 was set or
 .Fa \*(lp*errfunc\*(rp\*(lp\*(rp
 returned non-zero.
-.It Dv GLOB_LIMIT
-The flag
-.Dv GLOB_MAXPATH
-was provided, and the specified limit passed to
-.Fn glob
-in
-.Fa pglob\->gl_matchc
-was reached.
 .El
 .Pp
 The arguments
@@ -427,8 +426,8 @@ compatible with the exception
 that the flags
 .Dv GLOB_ALTDIRFUNC ,
 .Dv GLOB_BRACE ,
+.Dv GLOB_LIMIT ,
 .Dv GLOB_MAGCHAR ,
-.Dv GLOB_MAXPATH ,
 .Dv GLOB_NOMAGIC ,
 .Dv GLOB_QUOTE ,
 and

lib/libc/gen/glob.c

@@ -170,9 +170,11 @@ glob(pattern, flags, errfunc, pglob)
 		if (!(flags & GLOB_DOOFFS))
 			pglob->gl_offs = 0;
 	}
-	if (flags & GLOB_MAXPATH)
+	if (flags & GLOB_LIMIT) {
 		limit = pglob->gl_matchc;
-	else
+		if (limit == 0)
+			limit = ARG_MAX;
+	} else
 		limit = 0;
 	pglob->gl_flags = flags & ~GLOB_MAGCHAR;
 	pglob->gl_errfunc = errfunc;
@@ -687,8 +689,10 @@ globextend(path, pglob, limit)
 	char *copy;
 	const Char *p;
 
-	if (*limit && pglob->gl_pathc > *limit)
-		return (GLOB_LIMIT);
+	if (*limit && pglob->gl_pathc > *limit) {
+		errno = 0;
+		return (GLOB_NOSPACE);
+	}
 
 	newsize = sizeof(*pathv) * (2 + pglob->gl_pathc + pglob->gl_offs);
 	pathv = pglob->gl_pathv ?

libexec/ftpd/ftpd.c

@@ -2658,7 +2658,7 @@ send_file_list(whichf)
 
 		memset(&gl, 0, sizeof(gl));
 		gl.gl_matchc = MAXGLOBARGS;
-		flags |= GLOB_MAXPATH;
+		flags |= GLOB_LIMIT;
 		freeglob = 1;
 		if (glob(whichf, flags, 0, &gl)) {
 			reply(550, "not found");

libexec/ftpd/popen.c

@@ -108,7 +108,7 @@ ftpd_popen(program, type)
 
 		memset(&gl, 0, sizeof(gl));
 		gl.gl_matchc = MAXGLOBARGS;
-		flags |= GLOB_MAXPATH;
+		flags |= GLOB_LIMIT;
 		if (glob(argv[argc], flags, NULL, &gl))
 			gargv[gargc++] = strdup(argv[argc]);
 		else