Fix FTP_PROXY to use user@host[@port] for FTP proxy and eliminate

undocumented FTP_PROXY_USER
Make FTP file errors contian hostname and path.
Pass the FTP port to libftp.
Partially un-HTMLify error messages returned from server
Handle "HTTP NNN" instead of "HTTP/V.vv NNN" response sent by
	pre-HTTP/1.0 servers

Reviewed by:		wollman

usr.bin/fetch/fetch.1

@@ -1,4 +1,4 @@
-.\" $Id: fetch.1,v 1.15 1997/02/22 19:54:56 peter Exp $
+.\" $Id: fetch.1,v 1.16 1997/02/22 23:43:32 wosch Exp $
 .Dd July 2, 1996
 .Dt FETCH 1
 .Os FreeBSD 2.2
@@ -140,12 +140,12 @@ colon and a port number.
 .Pp
 The
 .Tn FTP
-proxy client specifies
-.Dq anonymous
-as its user name, and passes the remote user name and host as the
+proxy client passes the remote username, host and port as the
 .Tn FTP
-session's password, in the form
-.Dq Va remoteuser Ns Li \&@ Ns Va remotehost .
+session's username, in the form
+.Do Va remoteuser Ns Li \&@ Ns Va remotehost
+.Op Li \^@ Ns Va port
+.Dc .
 The
 .Tn HTTP
 proxy client simply passes the originally-requested URI to the remote
@@ -153,10 +153,6 @@ server in an
 .Tn HTTP
 .Dq Li GET
 request.  HTTP proxy authentication is not yet implemented.
-When multiple proxy protcols are configured,
-.Nm
-will prefer
-.Tn HTTP .
 .Sh HTTP AUTHENTICATION
 The
 .Tn HTTP
@@ -251,13 +247,21 @@ the password used for
 transfers (default
 .Dq Va yourname Ns Li \&@ Ns Va yourhost )
 .It Ev FTP_PROXY
-the address of a proxy server which understands
+the address (in the form
+.Do Va hostname Ns
+.Op Li : Ns Va port
+.Dc )
+of a proxy server which understands
 .Tn FTP
 .It Ev HTTP_AUTH
 defines authentication parameters for
 .Tn HTTP
 .It Ev HTTP_PROXY
-the address of a proxy server which understands
+the address (in the form
+.Do Va hostname Ns
+.Op Li : Ns Va port
+.Dc )
+of a proxy server which understands
 .Tn HTTP
 .It Ev HTTP_PROXY_AUTH
 defines authentication parameters for

usr.bin/fetch/ftp.c

@@ -26,7 +26,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: ftp.c,v 1.2 1997/02/05 19:59:12 wollman Exp $
+ *	$Id: ftp.c,v 1.3 1997/02/10 18:49:40 wollman Exp $
  */
 
 #include <sys/types.h>
@@ -136,8 +136,8 @@ ftp_parse(struct fetch_state *fs, const char *uri)
 	} else {
 		ftps->ftp_user = 0;
 		ftps->ftp_hostname = safe_strdup(hostname);
-		ftps->ftp_port = port;
 	}
+	ftps->ftp_port = port;
 
 	p = ftps->ftp_remote_file = percent_decode(p);
 	/* now p is the decoded version */
@@ -190,7 +190,7 @@ ftp_proxy_parse(struct fetch_state *fs, const char *uri)
 	char *hostname;
 	char *port;
 	const char *user;
-	char *newpass;
+	char *newuser;
 	unsigned portno;
 	struct ftp_state *ftps;
 
@@ -229,30 +229,29 @@ ftp_proxy_parse(struct fetch_state *fs, const char *uri)
 	}
 
 	ftps = fs->fs_proto;
-	if (ftps->ftp_port != 21) {
-		ftp_close(fs);
-		warnx("`%s': FTP proxy requires the use of the standard port",
-		      uri);
-		return EX_USAGE;
-	}
 
-	ftps->ftp_port = portno;
 	user = ftps->ftp_user ? ftps->ftp_user : "anonymous";
-	newpass = safe_malloc(strlen(ftps->ftp_user 
-				     ? ftps->ftp_user : "anonymous")
-			      + 1 + strlen(ftps->ftp_hostname) + 1);
+	/* user @ hostname [ @port ] \0 */
+	newuser = safe_malloc(strlen(user) + 1 + strlen(ftps->ftp_hostname)
+			      + ((ftps->ftp_port != 21) ? 6 : 0) + 1);
 
-	strcpy(newpass, user);
-	strcat(newpass, "@");
-	strcpy(newpass, ftps->ftp_hostname);
+	strcpy(newuser, user);
+	strcat(newuser, "@");
+	strcat(newuser, ftps->ftp_hostname);
+	if (ftps->ftp_port != 21) {
+		char numbuf[6];
+
+		snprintf(numbuf, sizeof(numbuf), "%d", ftps->ftp_port);
+		numbuf[sizeof(numbuf)-1] = '\0';
+		strcat(newuser, "@");
+		strcat(newuser, numbuf);
+	}
+	
+	ftps->ftp_port = portno;
 	free(ftps->ftp_hostname);
 	ftps->ftp_hostname = safe_strdup(hostname);
-	free(ftps->ftp_password);
-	ftps->ftp_password = newpass;
 	free(ftps->ftp_user);
-	ftps->ftp_user = getenv("FTP_PROXY_USER");
-	if (ftps->ftp_user)
-		ftps->ftp_user = safe_strdup(ftps->ftp_user);
+	ftps->ftp_user = newuser;
 	return 0;
 }
 
@@ -286,7 +285,7 @@ ftp_retrieve(struct fetch_state *fs)
 	ftp = ftpLogin(ftps->ftp_hostname, 
 		       (char *)(ftps->ftp_user ? ftps->ftp_user : "anonymous"),
 		       /* XXX ^^^^ bad API */
-		       ftps->ftp_password, 0, fs->fs_verbose > 1,
+		       ftps->ftp_password, ftps->ftp_port, fs->fs_verbose > 1,
 		       &status);
 	if (ftp == 0) {
 		warnx("%s: %s", ftps->ftp_hostname, 
@@ -334,8 +333,9 @@ ftp_retrieve(struct fetch_state *fs)
 	remote = ftpGet(ftp, ftps->ftp_remote_file, &seekloc);
 	if (remote == 0) {
 		if (ftpErrno(ftp)) {
-			warnx("%s: %s", ftps->ftp_hostname, 
-			      ftpErrString(ftpErrno(ftp)));
+			warnx("ftp://%s/%s: FTP error:",
+				ftps->ftp_hostname, ftps->ftp_remote_file);
+			warnx("%s", ftpErrString(ftpErrno(ftp)));
 			fclose(ftp);
 			return EX_IOERR;
 		} else {

usr.bin/fetch/http.c

@@ -26,7 +26,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: http.c,v 1.3 1997/02/05 19:59:14 wollman Exp $
+ *	$Id: http.c,v 1.4 1997/02/11 20:46:05 wollman Exp $
  */
 
 #include <sys/types.h>
@@ -356,6 +356,44 @@ http_redirect(struct fetch_state *fs, char *new, int permanent)
 	return rv;
 }
 
+/*
+ * Read HTML-formatted data from remote and display it on stderr.
+ * This is extremely incomplete, as all it does is delete anything
+ * between angle brackets.  However, this is usually good enough for
+ * error messages.
+ */
+static void
+html_display(FILE *remote)
+{
+	char *line;
+	int linelen;
+	int inbracket = 0;
+
+
+	while ((line = fgetln(remote, &linelen)) != 0) {
+		char *end = line + linelen;
+		char *p;
+		int content = 0;
+
+		for (p = line; p < end; p++) {
+			if (*p == '<' && !inbracket) {
+				fwrite(line, 1, (p - line),
+					stderr);
+				inbracket = 1;
+			}
+			if (!inbracket && !content &&
+				*p != '\n' && *p != '\r')
+				content = 1;
+			if (*p == '>' && inbracket) {
+				line = p + 1;
+				inbracket = 0;
+			}
+		}
+		if (content && line < end)
+			fwrite(line, 1, (end - line), stderr);
+	}
+}
+
 /*
  * Get a file using HTTP.  We will try to implement HTTP/1.1 eventually.
  * This subroutine makes heavy use of the 4.4-Lite standard I/O library,
@@ -379,6 +417,7 @@ http_retrieve(struct fetch_state *fs)
 	const char *env;
 	int timo;
 	char *line, *new_location;
+	char *errstr = 0;
 	size_t linelen, readresult, writeresult;
 	off_t total_length, restart_from;
 	time_t last_modified, when_to_retry;
@@ -567,7 +606,7 @@ got100reply:
 	 * to suck the entire file.  (It had better be, since
 	 * we used it to grab the first line.)
 	 */
-	if (linelen < 5 || strncasecmp(line, "http/", 5) != 0) {
+	if (linelen < 5 || strncasecmp(line, "http", 4) != 0) {
 		if (to_stdout)
 			local = fopen("/dev/stdout", "w");
 		else
@@ -630,8 +669,9 @@ got100reply:
 		break;
 	case 301:		/* Resource has moved permanently */
 		if (!fs->fs_auto_retry)
-			goto spewerror;
-		redirection = 301;
+			errstr = safe_strdup(line);
+		else
+			redirection = 301;
 		break;
 	case 302:		/* Resource has moved temporarily */
 		/*
@@ -646,37 +686,30 @@ got100reply:
 			unsetup_sigalrm();
 			return 0;
 		}
-		goto spewerror;
+		errstr = safe_strdup(line);
+		break;
 	case 401:		/* Unauthorized */
 		if (https->http_authentication)
-			goto spewerror;
-		autherror = 401;
+			errstr = safe_strdup(line);
+		else
+			autherror = 401;
 		break;
 	case 407:		/* Proxy Authentication Required */
 		if (https->http_proxy_authentication)
-			goto spewerror;
-		autherror = 407;
+			errstr = safe_strdup(line);
+		else
+			autherror = 407;
 		break;
 	case 503:		/* Service Unavailable */
 		if (!fs->fs_auto_retry)
-			goto spewerror;
-
-		retrying = 503;
+			errstr = safe_strdup(line);
+		else
+			retrying = 503;
 		break;
 
 	default:
-spewerror:
-		warnx("%s: %s: HTTP server returned error code %d", 
-		      fs->fs_outputfile, https->http_hostname, status);
-		if (fs->fs_verbose > 1) {
-			fputs(line, stderr);
-			fputc('\n', stderr);
-			while ((line = fgetln(remote, &linelen)) != 0)
-				fwrite(line, 1, linelen, stderr);
-		}
-		fclose(remote);
-		unsetup_sigalrm();
-		return EX_UNAVAILABLE;
+		errstr = safe_strdup(line);
+		break;
 	}
 
 	total_length = -1;	/* -1 means ``don't know'' */
@@ -801,7 +834,6 @@ doretry:
 	if (autherror == 407 && https->http_proxy_authentication)
 		goto doretry;
 	if (autherror) {
-		line = (char *)"HTTP/1.1 401 Unauthorized";
 		goto spewerror;
 	}
 
@@ -809,8 +841,7 @@ doretry:
 		int howlong;
 
 		if (when_to_retry == -1) {
-			/* This assignment is OK because all we do is print. */
-			line = (char *)"HTTP/1.1 503 Service Unavailable";
+			errstr = safe_strdup("HTTP/1.1 503 Service Unavailable");
 			goto spewerror;
 		}
 		howlong = when_to_retry - time(0);
@@ -823,6 +854,21 @@ doretry:
 		sleep(howlong);
 		goto doretry;
 	}
+
+	if (errstr != 0) {
+spewerror:
+		warnx("%s: %s: HTTP server returned error code %d", 
+		      fs->fs_outputfile, https->http_hostname, status);
+		if (fs->fs_verbose > 1) {
+			fputs(errstr, stderr);
+			fputc('\n', stderr);
+			html_display(remote);
+		}
+		free(errstr);
+		fclose(remote);
+		unsetup_sigalrm();
+		return EX_UNAVAILABLE;
+	}
 		
 	if (redirection && new_location) {
 		fclose(remote);
@@ -935,6 +981,8 @@ cantauth:
 /*
  * The format of the response line for an HTTP request is:
  *	HTTP/V.vv{WS}999{WS}Explanatory text for humans to read\r\n
+ * Old pre-HTTP/1.0 servers can return
+ *	HTTP{WS}999{WS}Explanatory text for humans to read\r\n
  * Where {WS} represents whitespace (spaces and/or tabs) and 999
  * is a machine-interprable result code.  We return the integer value
  * of that result code, or the impossible value `0' if we are unable to
@@ -946,15 +994,11 @@ http_first_line(const char *line)
 	char *ep;
 	unsigned long ul;
 
-	if (strncasecmp(line, "http/", 5) != 0)
+	if (strncasecmp(line, "http", 4) != 0)
 		return 0;
 
-	line += 5;
-	while (*line && isdigit(*line))	/* skip major version number */
-		line++;
-	if (*line++ != '.')	/* skip period */
-		return 0;
-	while (*line && isdigit(*line))	/* skip minor version number */
+	line += 4;
+	while (*line && !isspace(*line)) /* skip non-whitespace */
 		line++;
 	while (*line && isspace(*line))	/* skip first whitespace */
 		line++;