Deal with broken Web sites which return 302 responses rather than 404

and an error document when the requested resource does not exist.  Grrr.

Requested by:	asami

usr.bin/fetch/fetch.1

@@ -1,13 +1,13 @@
-.\" $Id: fetch.1,v 1.27 1999/01/15 16:56:22 wollman Exp $
-.Dd January 15, 1999
+.\" $Id: fetch.1,v 1.28 1999/02/03 20:43:28 fenner Exp $
+.Dd February 22, 1999
 .Dt FETCH 1
-.Os FreeBSD 3.1
+.Os FreeBSD 4.0
 .Sh NAME
 .Nm fetch
 .Nd retrieve a file by Uniform Resource Locator
 .Sh SYNOPSIS
 .Nm fetch
-.Op Fl MPablmnpqrtv
+.Op Fl AMPablmnpqrtv
 .Op Fl S Ar size
 .Op Fl T Ar timeout
 .Op Fl o Ar file
@@ -49,6 +49,10 @@ flags.
 .Pp
 The following options are available:
 .Bl -tag -width Fl
+.It Fl A
+Do not automatically follow ``temporary'' (302) redirects.  Some
+broken Web sites will return a redirect instead of a not-found error
+when the requested object does not exist.
 .It Fl a
 Automatically retry the transfer upon soft failures.
 .It Fl b
@@ -192,7 +196,7 @@ proxy client passes the remote username, host and port as the
 .Tn FTP
 session's username, in the form
 .Do Va remoteuser Ns Li \&@ Ns Va remotehost
-.Op Li \^@ Ns Va port
+.Op Li \&@ Ns Va port
 .Dc .
 The
 .Tn HTTP
@@ -254,7 +258,7 @@ or
 .Pp
 and
 .Nm
-will prompt for the missing information if it is required.  She might
+will prompt for any missing information when it is required.  She might
 also specify a realm of
 .Dq Li \&*
 instead of

usr.bin/fetch/http.c

@@ -26,7 +26,7 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: http.c,v 1.24 1999/01/15 17:10:31 wollman Exp $
+ *	$Id: http.c,v 1.25 1999/02/03 20:24:53 fenner Exp $
  */
 
 #include <sys/types.h>
@@ -703,7 +703,6 @@ http_retrieve(struct fetch_state *fs)
 	line[linelen - 1] = '\0'; /* turn line into a string */
 	status = http_first_line(line);
 
-	/* In the future, we might handle redirection and other responses. */
 	switch(status) {
 	case 100:		/* Continue */
 		goto got100reply;
@@ -715,17 +714,29 @@ http_retrieve(struct fetch_state *fs)
 		/* can only happen when restarting */
 		break;
 	case 301:		/* Resource has moved permanently */
-		if (!fs->fs_auto_retry)
+		if (fs->fs_auto_retry < 1)
 			errstr = safe_strdup(line);
 		else
 			redirection = 301;
 		break;
 	case 302:		/* Resource has moved temporarily */
 		/*
-		 * We don't test fs->fs_auto_retry here so that this
-		 * sort of redirection is transparent to the user.
+		 * We formerly didn't test fs->fs_auto_retry here,
+		 * so that this sort of redirection would be transparent
+		 * to the user.  Unfortunately, there are a lot of idiots
+		 * out there running Web sites, and some of them have
+		 * decided to implement the following stupidity: rather
+		 * than returning the correct `404 Not Found' error
+		 * when something is not found, they instead return
+		 * a 302 redirect, giving the erroneous impression that
+		 * the requested resource actually exists.  This
+		 * breaks any client which expects a non-existent resource
+		 * to elicit a 40x response.  Grrr.
 		 */
-		redirection = 302;
+		if (fs->fs_auto_retry < 0) /* -A flag */
+			errstr = safe_strdup(line);
+		else
+			redirection = 302;
 		break;
 	case 304:		/* Object is unmodified */
 		if (fs->fs_mirror) {

usr.bin/fetch/main.c

@@ -24,7 +24,7 @@
  * SUCH DAMAGE.
  */
 
-/* $Id: main.c,v 1.48 1998/11/08 23:18:48 des Exp $ */
+/* $Id: main.c,v 1.49 1998/12/08 13:00:49 cracauer Exp $ */
 
 #include <sys/types.h>
 
@@ -49,10 +49,11 @@ static sigjmp_buf sigbuf;
 static int get(struct fetch_state *volatile fs);
 
 static void
-usage()
+usage(void)
 {
-	fprintf(stderr, "%s\n%s\n", 
-		"usage: fetch [-DHILMNPRTVablmnpqrstv] [-o outputfile] [-S bytes]",
+	fprintf(stderr,
+		"usage: fetch [-ADHILMNPRTVablmnpqrstv] [-o outputfile] "
+		"[-S bytes]\n"
 		"             [-f file -h host [-c dir] | URL]");
 	exit(EX_USAGE);
 }
@@ -75,8 +76,13 @@ main(int argc, char *const *argv)
     fs.fs_expectedsize = -1;
     change_to_dir = file_to_get = hostname = 0;
 
-    while ((c = getopt(argc, argv, "abc:D:f:h:HIlLmMnNo:pPqRrS:stT:vV:")) != -1) {
+#define OPT_STRING	"Aabc:D:f:h:HIlLmMnNo:pPqRrS:stT:vV:"
+    while ((c = getopt(argc, argv, OPT_STRING)) != -1) {
 	    switch (c) {
+	    case 'A':
+		    fs.fs_auto_retry = -1;
+		    break;
+
 	    case 'D': case 'H': case 'I': case 'L': case 'N': case 'V': 
 		    break;	/* ncftp compatibility */