Reorganize some of the http code and split it into more functions.

Implement fetchStatHTTP().
Unbungle struct url, and add fetchFreeURL().
Document it.

lib/libfetch/fetch.3

@@ -28,11 +28,12 @@
 .Dt FETCH 3
 .Os
 .Sh NAME
+.Nm fetchParseURL ,
+.Nm fetchFreeURL ,
 .Nm fetchGetURL ,
 .Nm fetchPutURL ,
 .Nm fetchStatURL ,
 .Nm fetchListURL ,
-.Nm fetchParseURL ,
 .Nm fetchGet ,
 .Nm fetchPut ,
 .Nm fetchStat ,
@@ -56,6 +57,10 @@
 .Fd #include <sys/param.h>
 .Fd #include <stdio.h>
 .Fd #include <fetch.h>
+.Ft struct url *
+.Fn fetchParseURL "char *URL"
+.Ft void
+.Fn fetchFreeURL "struct url *URL"
 .Ft FILE *
 .Fn fetchGetURL "char *URL" "char *flags"
 .Ft FILE *
@@ -64,8 +69,6 @@
 .Fn fetchStatURL "char *URL" "struct url_stat *us" "char *flags"
 .Ft struct url_ent *
 .Fn fetchListURL "char *URL" "char *flags"
-.Ft struct url *
-.Fn fetchParseURL "char *URL"
 .Ft FILE *
 .Fn fetchGet "struct url *URL" "char *flags"
 .Ft FILE *
@@ -103,6 +106,25 @@
 These functions implement a high-level library for retrieving and
 uploading files using Uniform Resource Locators (URLs).
 .Pp
+.Fn fetchParseURL
+takes a URL in the form of a null-terminated string and splits it into
+its components function according to the Common Internet Scheme Syntax
+detailed in RFC1738. A regular expression which produces this syntax
+is:
+.Bd -literal
+    <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
+.Ed
+.Pp
+Note that some components of the URL are not necessarily relevant to
+all URL schemes.
+For instance, the file scheme only needs the <scheme>
+and <document> components.
+.Pp
+The pointer returned by
+.Fn fetchParseURL
+should be freed using
+.Fn fetchFreeURL .
+.Pp
 .Fn fetchGetURL
 and
 .Fn fetchPutURL
@@ -158,25 +180,6 @@ The pointer returned by
 should be freed using
 .Fn free .
 .Pp
-.Fn fetchParseURL
-takes a URL in the form of a null-terminated string and splits it into
-its components function according to the Common Internet Scheme Syntax
-detailed in RFC1738. A regular expression which produces this syntax
-is:
-.Bd -literal
-    <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
-.Ed
-.Pp
-Note that some components of the URL are not necessarily relevant to
-all URL schemes.
-For instance, the file scheme only needs the <scheme>
-and <document> components.
-.Pp
-The pointer returned by
-.Fn fetchParseURL
-should be freed using
-.Fn free .
-.Pp
 .Fn fetchGet ,
 .Fn fetchPut
 and
@@ -414,7 +417,6 @@ Some parts of the library are not yet implemented.
 The most notable
 examples of this are
 .Fn fetchPutHTTP ,
-.Fn fetchStatHTTP ,
 .Fn fetchListHTTP ,
 .Fn fetchListFTP
 and FTP proxy support.

lib/libfetch/fetch.c

@@ -152,7 +152,7 @@ fetchGetURL(char *URL, char *flags)
     
     f = fetchGet(u, flags);
     
-    free(u);
+    fetchFreeURL(u);
     return f;
 }
 
@@ -171,7 +171,7 @@ fetchPutURL(char *URL, char *flags)
     
     f = fetchPut(u, flags);
     
-    free(u);
+    fetchFreeURL(u);
     return f;
 }
 
@@ -189,7 +189,7 @@ fetchStatURL(char *URL, struct url_stat *us, char *flags)
 
     s = fetchStat(u, us, flags);
 
-    free(u);
+    fetchFreeURL(u);
     return s;
 }
 
@@ -207,7 +207,7 @@ fetchListURL(char *URL, char *flags)
 
     ue = fetchList(u, flags);
 
-    free(u);
+    fetchFreeURL(u);
     return ue;
 }
 
@@ -282,19 +282,13 @@ fetchParseURL(char *URL)
 
 nohost:
     /* document */
-    if (*p) {
+    if (!*p)
-	struct url *t;
+	p = "/";
-	t = realloc(u, sizeof *u + strlen(p) - 1);
+    
-	if (t == NULL) {
+    if ((u->doc = strdup(p)) == NULL) {
-	    errno = ENOMEM;
+	errno = ENOMEM;
-	    _fetch_syserr();
+	_fetch_syserr();
-	    goto ouch;
+	goto ouch;
-	}
-	u = t;
-	strcpy(u->doc, p);
-    } else {
-	u->doc[0] = '/';
-	u->doc[1] = 0;
     }
     
     DEBUG(fprintf(stderr,
@@ -313,3 +307,13 @@ fetchParseURL(char *URL)
     free(u);
     return NULL;
 }
+
+/*
+ * Free a URL
+ */
+void
+fetchFreeURL(struct url *u)
+{
+    free(u->doc);
+    free(u);
+}

lib/libfetch/fetch.h

@@ -40,14 +40,14 @@
 #define URL_PWDLEN 256
 
 struct url {
-    off_t	 offset;
-    size_t	 length;
     char	 scheme[URL_SCHEMELEN+1];
     char	 user[URL_USERLEN+1];
     char	 pwd[URL_PWDLEN+1];
     char	 host[MAXHOSTNAMELEN+1];
     int		 port;
-    char	 doc[2];
+    char	*doc;
+    off_t	 offset;
+    size_t	 length;
 };
 
 struct url_stat {
@@ -81,7 +81,6 @@ int		 fetchStatFTP(struct url *, struct url_stat *, char *);
 struct url_ent	*fetchListFTP(struct url *, char *);
 
 /* Generic functions */
-struct url	*fetchParseURL(char *);
 FILE		*fetchGetURL(char *, char *);
 FILE		*fetchPutURL(char *, char *);
 int		 fetchStatURL(char *, struct url_stat *, char *);
@@ -91,6 +90,10 @@ FILE		*fetchPut(struct url *, char *);
 int		 fetchStat(struct url *, struct url_stat *, char *);
 struct url_ent	*fetchList(struct url *, char *);
 
+/* URL parsing */
+struct url	*fetchParseURL(char *);
+void		 fetchFreeURL(struct url *);
+
 /* Last error code */
 extern int	 fetchLastErrCode;
 extern int	 fetchTimeout;

lib/libfetch/http.c

@@ -64,11 +64,13 @@
 
 #include <err.h>
 #include <ctype.h>
+#include <locale.h>
 #include <netdb.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <time.h>
 #include <unistd.h>
 
 #include "fetch.h"
@@ -292,25 +294,19 @@ _http_auth(char *usr, char *pwd)
 }
 
 /*
- * Retrieve a file by HTTP
+ * Connect to server or proxy
  */
 FILE *
-fetchGetHTTP(struct url *URL, char *flags)
+_http_connect(struct url *URL, char *flags)
 {
-    int sd = -1, e, i, enc = ENC_NONE, direct, verbose;
+    int direct, sd = -1, verbose;
-    struct cookie *c;
-    char *ln, *p, *px, *q;
-    FILE *f, *cf;
     size_t len;
-    off_t pos = 0;
+    char *px;
-
+    FILE *f;
+    
     direct = (flags && strchr(flags, 'd'));
     verbose = (flags && strchr(flags, 'v'));
     
-    /* allocate cookie */
-    if ((c = calloc(1, sizeof *c)) == NULL)
-	return NULL;
-
     /* check port */
     if (!URL->port) {
 	struct servent *se;
@@ -374,20 +370,40 @@ fetchGetHTTP(struct url *URL, char *flags)
     /* reopen as stream */
     if ((f = fdopen(sd, "r+")) == NULL)
 	goto ouch;
-    c->real_f = f;
+    
+    return f;
 
+ouch:
+    if (sd >= 0)
+	close(sd);
+    _http_seterr(999); /* XXX do this properly RSN */
+    return NULL;
+}
+
+/*
+ * Send a HEAD or GET request
+ */
+int
+_http_request(FILE *f, char *op, struct url *URL, char *flags)
+{
+    int e, verbose;
+    char *ln, *p;
+    size_t len;
+    
+    verbose = (flags && strchr(flags, 'v'));
+    
     /* send request (proxies require absolute form, so use that) */
     if (verbose)
 	_fetch_info("requesting http://%s:%d%s",
 		    URL->host, URL->port, URL->doc);
-    _http_cmd(f, "GET http://%s:%d%s HTTP/1.1" ENDL,
+    _http_cmd(f, "%s %s://%s:%d%s HTTP/1.1" ENDL,
-	      URL->host, URL->port, URL->doc);
+	      op, URL->scheme, URL->host, URL->port, URL->doc);
 
     /* start sending headers away */
     if (URL->user[0] || URL->pwd[0]) {
 	char *auth_str = _http_auth(URL->user, URL->pwd);
 	if (!auth_str)
-	    goto fouch;
+	    return 999; /* XXX wrong */
 	_http_cmd(f, "Authorization: Basic %s" ENDL, auth_str);
 	free(auth_str);
     }
@@ -399,7 +415,7 @@ fetchGetHTTP(struct url *URL, char *flags)
 
     /* get response */
     if ((ln = fgetln(f, &len)) == NULL)
-	goto fouch;
+	return 999;
     DEBUG(fprintf(stderr, "response: [\033[1m%*.*s\033[m]\n",
 		  (int)len-2, (int)len-2, ln));
     
@@ -410,9 +426,55 @@ fetchGetHTTP(struct url *URL, char *flags)
     while ((p < ln + len) && !isdigit(*p))
 	p++;
     if (!isdigit(*p))
-	goto fouch;
+	return 999;
+    
     e = atoi(p);
     DEBUG(fprintf(stderr, "code:     [\033[1m%d\033[m]\n", e));
+    return e;
+}
+
+/*
+ * Check a header line
+ */
+char *
+_http_match(char *str, char *hdr)
+{
+    while (*str && *hdr && tolower(*str++) == tolower(*hdr++))
+	/* nothing */;
+    if (*str || *hdr != ':')
+	return NULL;
+    while (*hdr && isspace(*++hdr))
+	/* nothing */;
+    return hdr;
+}
+
+/*
+ * Retrieve a file by HTTP
+ */
+FILE *
+fetchGetHTTP(struct url *URL, char *flags)
+{
+    int e, enc = ENC_NONE, i, verbose;
+    struct cookie *c;
+    char *ln, *p, *q;
+    FILE *f, *cf;
+    size_t len;
+    off_t pos = 0;
+
+    verbose = (flags && strchr(flags, 'v'));
+    
+    /* allocate cookie */
+    if ((c = calloc(1, sizeof *c)) == NULL)
+	return NULL;
+
+    /* connect */
+    if ((f = _http_connect(URL, flags)) == NULL) {
+	free(c);
+	return NULL;
+    }
+    c->real_f = f;
+
+    e = _http_request(f, "GET", URL, flags);
     
     /* add code to handle redirects later */
     if (e != (URL->offset ? HTTP_PARTIAL : HTTP_OK)) {
@@ -426,49 +488,33 @@ fetchGetHTTP(struct url *URL, char *flags)
 	    goto fouch;
 	if ((ln[0] == '\r') || (ln[0] == '\n'))
 	    break;
-	DEBUG(fprintf(stderr, "header:	 [\033[1m%*.*s\033[m]\n",
+	while (isspace(ln[len-1]))
-		      (int)len-2, (int)len-2, ln));
+	    --len;
-#define XFERENC "Transfer-Encoding:"
+	ln[len] = '\0'; /* XXX */
-	if (strncasecmp(ln, XFERENC, sizeof XFERENC - 1) == 0) {
+	DEBUG(fprintf(stderr, "header:	 [\033[1m%s\033[m]\n", ln));
-	    p = ln + sizeof XFERENC - 1;
+	if ((p = _http_match("Transfer-Encoding", ln)) != NULL) {
-	    while ((p < ln + len) && isspace(*p))
+	    for (q = p; *q && !isspace(*q); q++)
-		p++;
-	    for (q = p; (q < ln + len) && !isspace(*q); q++)
 		/* VOID */ ;
 	    *q = 0;
 	    if (strcasecmp(p, "chunked") == 0)
 		enc = ENC_CHUNKED;
-	    DEBUG(fprintf(stderr, "xferenc:  [\033[1m%s\033[m]\n", p));
+	    DEBUG(fprintf(stderr, "transfer encoding:  [\033[1m%s\033[m]\n", p));
-#undef XFERENC
+	} else if ((p = _http_match("Content-Type", ln)) != NULL) {
-#define CONTTYPE "Content-Type:"
+	    for (i = 0; *p && i < HTTPCTYPELEN; p++, i++)
-	} else if (strncasecmp(ln, CONTTYPE, sizeof CONTTYPE - 1) == 0) {
+		    c->content_type[i] = *p;
-	    p = ln + sizeof CONTTYPE - 1;
-	    while ((p < ln + len) && isspace(*p))
-		p++;
-	    for (i = 0; p < ln + len; p++)
-		if (i < HTTPCTYPELEN)
-		    c->content_type[i++] = *p;
 	    do c->content_type[i--] = 0; while (isspace(c->content_type[i]));
-	    DEBUG(fprintf(stderr, "conttype: [\033[1m%s\033[m]\n",
+	    DEBUG(fprintf(stderr, "content type: [\033[1m%s\033[m]\n",
 			  c->content_type));
-#undef CONTTYPE
+	} else if ((p = _http_match("Content-Range", ln)) != NULL) {
-#define CONTRANGE "Content-Range:"
+	    if (strncasecmp(p, "bytes ", 6) != 0)
-#define BYTES "bytes "
-	} else if (strncasecmp(ln, CONTRANGE, sizeof CONTRANGE - 1) == 0) {
-	    p = ln + sizeof CONTRANGE - 1;
-	    while ((p < ln + len) && isspace(*p))
-		p++;
-	    if (strncasecmp(p, BYTES, sizeof BYTES - 1) != 0
-		|| (p += 6) >= ln + len)
 		goto fouch;
-	    while ((p < ln + len) && isdigit(*p))
+	    p += 6;
+	    while (*p && isdigit(*p))
 		pos = pos * 10 + (*p++ - '0');
 	    /* XXX wouldn't hurt to be slightly more paranoid here */
-	    DEBUG(fprintf(stderr, "contrange: [\033[1m%lld-\033[m]\n", pos));
+	    DEBUG(fprintf(stderr, "content range: [\033[1m%lld-\033[m]\n", pos));
 	    if (pos > URL->offset)
 		goto fouch;
-#undef BYTES
-#undef CONTRANGE
 	}
     }
 
@@ -488,12 +534,6 @@ fetchGetHTTP(struct url *URL, char *flags)
 		
     return cf;
     
-ouch:
-    if (sd >= 0)
-	close(sd);
-    free(c);
-    _http_seterr(999); /* XXX do this properly RSN */
-    return NULL;
 fouch:
     fclose(f);
     free(c);
@@ -516,10 +556,61 @@ fetchPutHTTP(struct url *URL, char *flags)
  * Get an HTTP document's metadata
  */
 int
-fetchStatHTTP(struct url *url, struct url_stat *us, char *flags)
+fetchStatHTTP(struct url *URL, struct url_stat *us, char *flags)
 {
-    warnx("fetchStatHTTP(): not implemented");
+    int e, verbose;
-    return -1;
+    size_t len;
+    char *ln, *p;
+    FILE *f;
+    
+    verbose = (flags && strchr(flags, 'v'));
+    
+    /* connect */
+    if ((f = _http_connect(URL, flags)) == NULL)
+	return -1;
+
+    if ((e = _http_request(f, "HEAD", URL, flags)) != HTTP_OK) {
+	_http_seterr(e);
+	goto ouch;
+    }
+
+    while (1) {
+	if ((ln = fgetln(f, &len)) == NULL)
+	    goto fouch;
+	if ((ln[0] == '\r') || (ln[0] == '\n'))
+	    break;
+	while (isspace(ln[len-1]))
+	    --len;
+	ln[len] = '\0'; /* XXX */
+	DEBUG(fprintf(stderr, "header:	 [\033[1m%s\033[m]\n", ln));
+	if ((p = _http_match("Last-Modified", ln)) != NULL) {
+	    struct tm tm;
+	    char locale[64];
+
+	    strncpy(locale, setlocale(LC_TIME, NULL), sizeof locale);
+	    setlocale(LC_TIME, "C");
+	    strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+	    /* XXX should add support for date-2 and date-3 */
+	    setlocale(LC_TIME, locale);
+	    us->atime = us->mtime = timegm(&tm);
+	    DEBUG(fprintf(stderr, "last modified: [\033[1m%04d-%02d-%02d "
+			  "%02d:%02d:%02d\033[m]\n",
+			  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+			  tm.tm_hour, tm.tm_min, tm.tm_sec));
+	} else if ((p = _http_match("Content-Length", ln)) != NULL) {
+	    us->size = 0;
+	    while (*p && isdigit(*p))
+		us->size = us->size * 10 + (*p++ - '0');
+	    DEBUG(fprintf(stderr, "content length: [\033[1m%lld\033[m]\n", us->size));
+	}
+    }
+    
+    return 0;
+ ouch:
+    _http_seterr(999); /* XXX do this properly RSN */
+ fouch:
+    fclose(f);
+    return -1;    
 }
 
 /*