Add support for HTTP 1.1 If-Modified-Since behavior.

fetch(1) accepts a new argument -i <file> that if specified will cause
the file to be downloaded only if it is more recent than the mtime of
<file>.

libfetch(3) accepts the mtime in the url structure and a flag to
indicate when this behavior is desired.

PR:		bin/87841
Submitted by:	Jukka A. Ukkonen <jau@iki.fi> (partially)
Reviewed by:	des, ru
MFC after:	3 weeks

lib/libfetch/fetch.3

@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 18, 2007
+.Dd December 14, 2008
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -165,9 +165,16 @@ struct url {
     char	*doc;
     off_t	 offset;
     size_t	 length;
+    time_t	 ims_time;
 };
 .Ed
 .Pp
+The
+.Va ims_time
+field stores the time value for
+.Li If-Modified-Since
+HTTP requests.
+.Pp
 The pointer returned by
 .Fn fetchMakeURL
 or
@@ -353,6 +360,22 @@ and
 .Fn fetchPutHTTP
 will use a direct connection even if a proxy server is defined.
 .Pp
+If the
+.Ql i
+(if-modified-since) flag is specified, and
+the
+.Va ims_time
+field is set in 
+.Vt "struct url" ,
+then
+.Fn fetchXGetHTTP
+and
+.Fn fetchGetHTTP
+will send a conditional
+.Li If-Modified-Since
+HTTP header to only fetch the content if it is newer than
+.Va ims_time .
+.Pp
 Since there seems to be no good way of implementing the HTTP PUT
 method in a manner consistent with the rest of the
 .Nm fetch

lib/libfetch/fetch.h

@@ -46,6 +46,7 @@ struct url {
 	char		*doc;
 	off_t		 offset;
 	size_t		 length;
+	time_t		 ims_time;
 };
 
 struct url_stat {

lib/libfetch/http.c

@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 
 #include <ctype.h>
 #include <err.h>
@@ -92,6 +93,7 @@ __FBSDID("$FreeBSD$");
 #define HTTP_MOVED_PERM		301
 #define HTTP_MOVED_TEMP		302
 #define HTTP_SEE_OTHER		303
+#define HTTP_NOT_MODIFIED	304
 #define HTTP_TEMP_REDIRECT	307
 #define HTTP_NEED_AUTH		401
 #define HTTP_NEED_PROXY_AUTH	407
@@ -797,20 +799,23 @@ FILE *
 http_request(struct url *URL, const char *op, struct url_stat *us,
     struct url *purl, const char *flags)
 {
+	char timebuf[80];
+	char hbuf[MAXHOSTNAMELEN + 7], *host;
 	conn_t *conn;
 	struct url *url, *new;
-	int chunked, direct, need_auth, noredirect, verbose;
+	int chunked, direct, ims, need_auth, noredirect, verbose;
 	int e, i, n, val;
 	off_t offset, clength, length, size;
 	time_t mtime;
 	const char *p;
 	FILE *f;
 	hdr_t h;
-	char hbuf[MAXHOSTNAMELEN + 7], *host;
+	struct tm *timestruct;
 
 	direct = CHECK_FLAG('d');
 	noredirect = CHECK_FLAG('A');
 	verbose = CHECK_FLAG('v');
+	ims = CHECK_FLAG('i');
 
 	if (direct && purl) {
 		fetchFreeURL(purl);
@@ -879,6 +884,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			    op, url->doc);
 		}
 
+		if (ims && url->ims_time) {
+			timestruct = gmtime((time_t *)&url->ims_time);
+			(void)strftime(timebuf, 80, "%a, %d %b %Y %T GMT",
+			    timestruct);
+			if (verbose)
+				fetch_info("If-Modified-Since: %s", timebuf);
+			http_cmd(conn, "If-Modified-Since: %s", timebuf);
+		}
 		/* virtual host */
 		http_cmd(conn, "Host: %s", host);
 
@@ -940,6 +953,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		switch (http_get_reply(conn)) {
 		case HTTP_OK:
 		case HTTP_PARTIAL:
+		case HTTP_NOT_MODIFIED:
 			/* fine */
 			break;
 		case HTTP_MOVED_PERM:
@@ -1074,7 +1088,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		}
 
 		/* we have a hit or an error */
-		if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))
+		if (conn->err == HTTP_OK
+		    || conn->err == HTTP_NOT_MODIFIED
+		    || conn->err == HTTP_PARTIAL
+		    || HTTP_ERROR(conn->err))
 			break;
 
 		/* all other cases: we got a redirect */
@@ -1102,6 +1119,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		  (long long)offset, (long long)length,
 		  (long long)size, (long long)clength));
 
+	if (conn->err == HTTP_NOT_MODIFIED) {
+		http_seterr(HTTP_NOT_MODIFIED);
+		return (NULL);
+	}
+
 	/* check for inconsistencies */
 	if (clength != -1 && length != -1 && clength != length) {
 		http_seterr(HTTP_PROTOCOL_ERROR);

usr.bin/fetch/fetch.1

@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 11, 2003
+.Dd December 14, 2008
 .Dt FETCH 1
 .Os
 .Sh NAME
@@ -39,6 +39,7 @@
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
 .Op Fl B Ar bytes
+.Op Fl i Ar file
 .Op Fl N Ar file
 .Op Fl o Ar file
 .Op Fl S Ar bytes
@@ -48,6 +49,7 @@
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
 .Op Fl B Ar bytes
+.Op Fl i Ar file
 .Op Fl N Ar file
 .Op Fl o Ar file
 .Op Fl S Ar bytes
@@ -116,6 +118,12 @@ The file to retrieve is located on the host
 .Ar host .
 This option is deprecated and is provided for backward compatibility
 only.
+.It Fl i Ar file
+If-Modified-Since mode: the remote file will only be retrieved if it
+is newer than
+.Ar file
+on the local host.
+(HTTP only)
 .It Fl l
 If the target is a file-scheme URL, make a symbolic link to the target
 rather than trying to copy it.
@@ -249,6 +257,12 @@ If multiple URLs are listed on the command line,
 .Nm
 will attempt to retrieve each one of them in turn, and will return
 zero only if they were all successfully retrieved.
+.Pp
+If the
+.Fl i
+argument is used and the remote file is not newer than the
+specified file then the command will still return success,
+although no file is transferred.
 .Sh SEE ALSO
 .Xr fetch 3
 .Sh HISTORY

usr.bin/fetch/fetch.c

@@ -60,6 +60,8 @@ int	 d_flag;	/*    -d: direct connection */
 int	 F_flag;	/*    -F: restart without checking mtime  */
 char	*f_filename;	/*    -f: file to fetch */
 char	*h_hostname;	/*    -h: host to fetch from */
+int	 i_flag;	/*    -i: specify input file for mtime comparison */
+char	*i_filename;	/*        name of input file */
 int	 l_flag;	/*    -l: link rather than copy file: URLs */
 int	 m_flag;	/* -[Mm]: mirror mode */
 char	*N_filename;	/*    -N: netrc file name */
@@ -382,6 +384,14 @@ fetch(char *URL, const char *path)
 		if (A_flag)
 			strcat(flags, "A");
 		timeout = T_secs ? T_secs : http_timeout;
+		if (i_flag) {
+			if (stat(i_filename, &sb)) {
+				warn("%s: stat()", i_filename);
+				goto failure;
+			}
+			url->ims_time = sb.st_mtime;
+			strcat(flags, "i");
+		}
 	}
 
 	/* set the protocol timeout. */
@@ -449,7 +459,14 @@ fetch(char *URL, const char *path)
 		goto signal;
 	if (f == NULL) {
 		warnx("%s: %s", URL, fetchLastErrString);
-		goto failure;
+		if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0
+		    && fetchLastErrCode == FETCH_OK
+		    && strcmp(fetchLastErrString, "Not Modified") == 0) {
+			/* HTTP Not Modified Response, return OK. */
+			r = 0;
+			goto done;
+		} else
+			goto failure;
 	}
 	if (sigint)
 		goto signal;
@@ -713,9 +730,9 @@ usage(void)
 {
 	fprintf(stderr, "%s\n%s\n%s\n%s\n",
 "usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
-"       [-T seconds] [-w seconds] URL ...",
+"       [-T seconds] [-w seconds] [-i file] URL ...",
 "       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]",
-"       [-T seconds] [-w seconds] -h host -f file [-c dir]");
+"       [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]");
 }
 
 
@@ -732,7 +749,7 @@ main(int argc, char *argv[])
 	int c, e, r;
 
 	while ((c = getopt(argc, argv,
-	    "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
+	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
 		switch (c) {
 		case '1':
 			once_flag = 1;
@@ -777,6 +794,10 @@ main(int argc, char *argv[])
 		case 'h':
 			h_hostname = optarg;
 			break;
+		case 'i':
+			i_flag = 1;
+			i_filename = optarg;
+			break;
 		case 'l':
 			l_flag = 1;
 			break;