New file descriptor allocation code, derived from similar code introduced

in OpenBSD by Niels Provos. The patch introduces a bitmap of allocated file descriptors which is used to locate available descriptors when a new one is needed. It also moves the task of growing the file descriptor table out of fdalloc(), reducing complexity in both fdalloc() and do_dup(). Debts of gratitude are owed to tjr@ (who provided the original patch on which this work is based), grog@ (for the gdb(4) man page) and rwatson@ (for assistance with pxeboot(8)).
svn path=/head/; revision=124548
2024-12-15 10:17:20 +00:00 · 2004-01-15 10:15:04 +00:00 · 2004-01-15 10:15:04 +00:00 · a2fe44e8cf · 2020-12-20 02:59:44 +00:00
commit a2fe44e8cf
parent 0161ff2670
8 changed files with 317 additions and 255 deletions
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@ -406,6 +406,7 @@ proc0_init(void *dummy __unused)
 	siginit(&proc0);

 	/* Create the file descriptor table. */
+	/* XXX this duplicates part of fdinit() */
 	fdp = &filedesc0;
 	p->p_fd = &fdp->fd_fd;
 	p->p_fdtol = NULL;
@ -415,6 +416,7 @@ proc0_init(void *dummy __unused)
 	fdp->fd_fd.fd_ofiles = fdp->fd_dfiles;
 	fdp->fd_fd.fd_ofileflags = fdp->fd_dfileflags;
 	fdp->fd_fd.fd_nfiles = NDFILE;
+	fdp->fd_fd.fd_map = fdp->fd_dmap;

 	/* Create the limits structures. */
 	p->p_limit = &limit0;
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_compat.h"

 #include <sys/param.h>
+#include <sys/limits.h>
 #include <sys/systm.h>
 #include <sys/syscallsubr.h>
 #include <sys/sysproto.h>
@ -96,6 +97,9 @@ enum dup_type { DUP_VARIABLE, DUP_FIXED };

 static int do_dup(struct thread *td, enum dup_type type, int old, int new,
    register_t *retval);
+static int	fd_first_free(struct filedesc *, int, int);
+static int	fd_last_used(struct filedesc *, int, int);
+static void	fdgrowtable(struct filedesc *, int);

 /*
 * Descriptor management.
@ -105,6 +109,102 @@ int nfiles;			/* actual number of open files */
 struct sx filelist_lock;	/* sx to protect filelist */
 struct mtx sigio_lock;		/* mtx to protect pointers to sigio */

+/*
+ * Find the first zero bit in the given bitmap, starting at low and not
+ * exceeding size - 1.
+ */
+static int
+fd_first_free(struct filedesc *fdp, int low, int size)
+{
+	NDSLOTTYPE *map = fdp->fd_map;
+	NDSLOTTYPE mask;
+	int off, maxoff;
+
+	if (low >= size)
+		return (low);
+
+	off = NDSLOT(low);
+	if (low % NDENTRIES) {
+		mask = ~(~(NDSLOTTYPE)0 >> (NDENTRIES - (low % NDENTRIES)));
+		if ((mask &= ~map[off]) != 0)
+			return (off * NDENTRIES + ffsl(mask) - 1);
+		++off;
+	}
+	for (maxoff = NDSLOTS(size); off < maxoff; ++off)
+		if ((mask = ~map[off]) != 0)
+			return (off * NDENTRIES + ffsl(mask) - 1);
+	return (size);
+}
+
+/*
+ * Find the highest non-zero bit in the given bitmap, starting at low and
+ * not exceeding size - 1.
+ */
+static int
+fd_last_used(struct filedesc *fdp, int low, int size)
+{
+	NDSLOTTYPE *map = fdp->fd_map;
+	NDSLOTTYPE mask;
+	int off, minoff;
+
+	if (low >= size)
+		return (-1);
+
+	off = NDSLOT(size);
+	if (size % NDENTRIES) {
+		mask = ~(~(NDSLOTTYPE)0 << (size % NDENTRIES));
+		if ((mask &= map[off]) != 0)
+			return (off * NDENTRIES + flsl(mask) - 1);
+		--off;
+	}
+	for (minoff = NDSLOT(low); off >= minoff; --off)
+		if ((mask = map[off]) != 0)
+			return (off * NDENTRIES + flsl(mask) - 1);
+	return (size - 1);
+}
+
+static int
+fdisused(struct filedesc *fdp, int fd)
+{
+        KASSERT(fd >= 0 && fd < fdp->fd_nfiles,
+            ("file descriptor %d out of range (0, %d)", fd, fdp->fd_nfiles));
+	return ((fdp->fd_map[NDSLOT(fd)] & NDBIT(fd)) != 0);
+}
+
+/*
+ * Mark a file descriptor as used.
+ */
+void
+fdused(struct filedesc *fdp, int fd)
+{
+	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
+	KASSERT(!fdisused(fdp, fd),
+	    ("fd already used"));
+	fdp->fd_map[NDSLOT(fd)] |= NDBIT(fd);
+	if (fd > fdp->fd_lastfile)
+		fdp->fd_lastfile = fd;
+	if (fd == fdp->fd_freefile)
+		fdp->fd_freefile = fd_first_free(fdp, fd, fdp->fd_nfiles);
+}
+
+/*
+ * Mark a file descriptor as unused.
+ */
+void
+fdunused(struct filedesc *fdp, int fd)
+{
+	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
+	KASSERT(fdisused(fdp, fd),
+	    ("fd is already unused"));
+	KASSERT(fdp->fd_ofiles[fd] == NULL,
+	    ("fd is still in use"));
+	fdp->fd_map[NDSLOT(fd)] &= ~NDBIT(fd);
+	if (fd < fdp->fd_freefile)
+		fdp->fd_freefile = fd;
+	if (fd == fdp->fd_lastfile)
+		fdp->fd_lastfile = fd_last_used(fdp, 0, fd);
+}
+
 /*
 * System calls on descriptors.
 */
@ -458,8 +558,10 @@ do_dup(td, type, old, new, retval)
 	struct proc *p;
 	struct file *fp;
 	struct file *delfp;
-	int error, newfd;
-	int holdleaders;
+	int error, holdleaders, maxfd;
+
+	KASSERT((type == DUP_VARIABLE || type == DUP_FIXED),
+	    ("invalid dup type %d", type));

 	p = td->td_proc;
 	fdp = p->p_fd;
@ -468,9 +570,12 @@ do_dup(td, type, old, new, retval)
 	 * Verify we have a valid descriptor to dup from and possibly to
 	 * dup to.
 	 */
-	if (old < 0 || new < 0 || new >= p->p_rlimit[RLIMIT_NOFILE].rlim_cur ||
-	    new >= maxfilesperproc)
+	if (old < 0 || new < 0)
 		return (EBADF);
+	maxfd = min((int)p->p_rlimit[RLIMIT_NOFILE].rlim_cur, maxfilesperproc);
+	if (new >= maxfd)
+		return (EMFILE);
+
 	FILEDESC_LOCK(fdp);
 	if (old >= fdp->fd_nfiles || fdp->fd_ofiles[old] == NULL) {
 		FILEDESC_UNLOCK(fdp);
@ -485,19 +590,26 @@ do_dup(td, type, old, new, retval)
 	fhold(fp);

 	/*
-	 * Expand the table for the new descriptor if needed.  This may
-	 * block and drop and reacquire the filedesc lock.
+	 * If the caller specified a file descriptor, make sure the file
+	 * table is large enough to hold it, and grab it.  Otherwise, just
+	 * allocate a new descriptor the usual way.  Since the filedesc
+	 * lock may be temporarily dropped in the process, we have to look
+	 * out for a race.
 	 */
-	if (type == DUP_VARIABLE || new >= fdp->fd_nfiles) {
-		error = fdalloc(td, new, &newfd);
-		if (error) {
+	if (type == DUP_FIXED) {
+		if (new >= fdp->fd_nfiles)
+			fdgrowtable(fdp, new + 1);
+		KASSERT(new < fdp->fd_nfiles,
+		    ("fdgrowtable() failed to grow table"));
+		if (fdp->fd_ofiles[new] == NULL)
+			fdused(fdp, new);
+	} else {
+		if ((error = fdalloc(td, &new)) != 0) {
 			FILEDESC_UNLOCK(fdp);
 			fdrop(fp, td);
 			return (error);
 		}
 	}
-	if (type == DUP_VARIABLE)
-		new = newfd;

 	/*
 	 * If the old file changed out from under us then treat it as a
@ -505,39 +617,38 @@ do_dup(td, type, old, new, retval)
 	 * avoid this case.
 	 */
 	if (fdp->fd_ofiles[old] != fp) {
-		if (fdp->fd_ofiles[new] == NULL) {
-			if (new < fdp->fd_freefile)
-				fdp->fd_freefile = new;
-			while (fdp->fd_lastfile > 0 &&
-			    fdp->fd_ofiles[fdp->fd_lastfile] == NULL)
-				fdp->fd_lastfile--;
-		}
+		/* we've allocated a descriptor which we won't use */
+		if (fdp->fd_ofiles[new] == NULL)
+			fdunused(fdp, new);
 		FILEDESC_UNLOCK(fdp);
 		fdrop(fp, td);
 		return (EBADF);
 	}
-	KASSERT(old != new, ("new fd is same as old"));
+	KASSERT(old != new,
+	    ("new fd is same as old"));

 	/*
-	 * Save info on the descriptor being overwritten.  We have
-	 * to do the unmap now, but we cannot close it without
-	 * introducing an ownership race for the slot.
+	 * Save info on the descriptor being overwritten.  We cannot close
+	 * it without introducing an ownership race for the slot, since we
+	 * need to drop the filedesc lock to call closef().
+	 *
+	 * XXX this duplicates parts of close().
 	 */
 	delfp = fdp->fd_ofiles[new];
-	if (delfp != NULL && p->p_fdtol != NULL) {
-		/*
-		 * Ask fdfree() to sleep to ensure that all relevant
-		 * process leaders can be traversed in closef().
-		 */
-		fdp->fd_holdleaderscount++;
-		holdleaders = 1;
-	} else
-		holdleaders = 0;
-	KASSERT(delfp == NULL || type == DUP_FIXED,
-	    ("dup() picked an open file"));
+	holdleaders = 0;
+	if (delfp != NULL) {
+		if (td->td_proc->p_fdtol != NULL) {
+			/*
+			 * Ask fdfree() to sleep to ensure that all relevant
+			 * process leaders can be traversed in closef().
+			 */
+			fdp->fd_holdleaderscount++;
+			holdleaders = 1;
+		}
+	}

 	/*
-	 * Duplicate the source descriptor, update lastfile
+	 * Duplicate the source descriptor
 	 */
 	fdp->fd_ofiles[new] = fp;
 	fdp->fd_ofileflags[new] = fdp->fd_ofileflags[old] &~ UF_EXCLOSE;
@ -550,8 +661,11 @@ do_dup(td, type, old, new, retval)
 	 * If we dup'd over a valid file, we now own the reference to it
 	 * and must dispose of it using closef() semantics (as if a
 	 * close() were performed on it).
+	 *
+	 * XXX this duplicates parts of close().
 	 */
-	if (delfp) {
+	if (delfp != NULL) {
+		/* XXX need to call knote_fdclose() */
 		mtx_lock(&Giant);
 		(void) closef(delfp, td);
 		mtx_unlock(&Giant);
@ -823,11 +937,12 @@ close(td, uap)
 	if ((unsigned)fd >= fdp->fd_nfiles ||
 	    (fp = fdp->fd_ofiles[fd]) == NULL) {
 		FILEDESC_UNLOCK(fdp);
-		error = EBADF;
-		goto done2;
+		mtx_unlock(&Giant);
+		return (EBADF);
 	}
 	fdp->fd_ofiles[fd] = NULL;
 	fdp->fd_ofileflags[fd] = 0;
+	fdunused(fdp, fd);
 	if (td->td_proc->p_fdtol != NULL) {
 		/*
 		 * Ask fdfree() to sleep to ensure that all relevant
@ -841,10 +956,6 @@ close(td, uap)
 	 * we now hold the fp reference that used to be owned by the descriptor
 	 * array.
 	 */
-	while (fdp->fd_lastfile > 0 && fdp->fd_ofiles[fdp->fd_lastfile] == NULL)
-		fdp->fd_lastfile--;
-	if (fd < fdp->fd_freefile)
-		fdp->fd_freefile = fd;
 	if (fd < fdp->fd_knlistsize) {
 		FILEDESC_UNLOCK(fdp);
 		knote_fdclose(td, fd);
@ -852,7 +963,6 @@ close(td, uap)
 		FILEDESC_UNLOCK(fdp);

 	error = closef(fp, td);
-done2:
 	mtx_unlock(&Giant);
 	if (holdleaders) {
 		FILEDESC_LOCK(fdp);
@ -1030,98 +1140,112 @@ fpathconf(td, uap)
 }

 /*
- * Allocate a file descriptor for the process.
+ * Grow the file table to accomodate (at least) nfd descriptors.  This may
+ * block and drop the filedesc lock, but it will reacquire it before
+ * returing.
 */
-static int fdexpand;
-SYSCTL_INT(_debug, OID_AUTO, fdexpand, CTLFLAG_RD, &fdexpand, 0, "");
-
-int
-fdalloc(td, want, result)
-	struct thread *td;
-	int want;
-	int *result;
+static void
+fdgrowtable(struct filedesc *fdp, int nfd)
 {
-	struct proc *p = td->td_proc;
-	struct filedesc *fdp = td->td_proc->p_fd;
-	int i;
-	int lim, last, nfiles;
-	struct file **newofile, **oldofile;
-	char *newofileflags;
+	struct file **ntable;
+	char *nfileflags;
+	int nnfiles, onfiles;
+	NDSLOTTYPE *nmap;

 	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);

+	KASSERT(fdp->fd_nfiles > 0,
+	    ("zero-length file table"));
+
+	/* compute the size of the new table */
+	onfiles = fdp->fd_nfiles;
+	nnfiles = NDSLOTS(nfd) * NDENTRIES; /* round up */
+	if (nnfiles <= onfiles)
+		/* the table is already large enough */
+		return;
+
+	/* allocate a new table and (if required) new bitmaps */
+	FILEDESC_UNLOCK(fdp);
+	MALLOC(ntable, struct file **, nnfiles * OFILESIZE,
+	    M_FILEDESC, M_ZERO | M_WAITOK);
+	nfileflags = (char *)&ntable[nnfiles];
+	if (NDSLOTS(nnfiles) > NDSLOTS(onfiles))
+		MALLOC(nmap, NDSLOTTYPE *, NDSLOTS(nnfiles) * NDSLOTSIZE,
+		    M_FILEDESC, M_ZERO | M_WAITOK);
+	else
+		nmap = NULL;
+	FILEDESC_LOCK(fdp);
+
 	/*
-	 * Search for a free descriptor starting at the higher
-	 * of want or fd_freefile.  If that fails, consider
-	 * expanding the ofile array.
+	 * We now have new tables ready to go.  Since we dropped the
+	 * filedesc lock to call malloc(), watch out for a race.
 	 */
-	lim = min((int)p->p_rlimit[RLIMIT_NOFILE].rlim_cur, maxfilesperproc);
-	for (;;) {
-		last = min(fdp->fd_nfiles, lim);
-		i = max(want, fdp->fd_freefile);
-		for (; i < last; i++) {
-			if (fdp->fd_ofiles[i] == NULL) {
-				fdp->fd_ofileflags[i] = 0;
-				if (i > fdp->fd_lastfile)
-					fdp->fd_lastfile = i;
-				if (want <= fdp->fd_freefile)
-					fdp->fd_freefile = i;
-				*result = i;
-				return (0);
-			}
-		}
-
-		/*
-		 * No space in current array.  Expand?
-		 */
-		if (i >= lim)
-			return (EMFILE);
-		if (fdp->fd_nfiles < NDEXTENT)
-			nfiles = NDEXTENT;
-		else
-			nfiles = 2 * fdp->fd_nfiles;
-		while (nfiles < want)
-			nfiles <<= 1;
-		FILEDESC_UNLOCK(fdp);
-		newofile = malloc(nfiles * OFILESIZE, M_FILEDESC, M_WAITOK);
-
-		/*
-		 * Deal with file-table extend race that might have
-		 * occurred while filedesc was unlocked.
-		 */
-		FILEDESC_LOCK(fdp);
-		if (fdp->fd_nfiles >= nfiles) {
-			FILEDESC_UNLOCK(fdp);
-			free(newofile, M_FILEDESC);
-			FILEDESC_LOCK(fdp);
-			continue;
-		}
-		newofileflags = (char *) &newofile[nfiles];
-		/*
-		 * Copy the existing ofile and ofileflags arrays
-		 * and zero the new portion of each array.
-		 */
-		i = fdp->fd_nfiles * sizeof(struct file *);
-		bcopy(fdp->fd_ofiles, newofile,	i);
-		bzero((char *)newofile + i,
-		    nfiles * sizeof(struct file *) - i);
-		i = fdp->fd_nfiles * sizeof(char);
-		bcopy(fdp->fd_ofileflags, newofileflags, i);
-		bzero(newofileflags + i, nfiles * sizeof(char) - i);
-		if (fdp->fd_nfiles > NDFILE)
-			oldofile = fdp->fd_ofiles;
-		else
-			oldofile = NULL;
-		fdp->fd_ofiles = newofile;
-		fdp->fd_ofileflags = newofileflags;
-		fdp->fd_nfiles = nfiles;
-		fdexpand++;
-		if (oldofile != NULL) {
-			FILEDESC_UNLOCK(fdp);
-			free(oldofile, M_FILEDESC);
-			FILEDESC_LOCK(fdp);
-		}
+	onfiles = fdp->fd_nfiles;
+	if (onfiles >= nnfiles) {
+		/* we lost the race, but that's OK */
+		free(ntable, M_FILEDESC);
+		if (nmap != NULL)
+			free(nmap, M_FILEDESC);
+		return;
 	}
+	bcopy(fdp->fd_ofiles, ntable, onfiles * sizeof(*ntable));
+	bcopy(fdp->fd_ofileflags, nfileflags, onfiles);
+	if (onfiles > NDFILE)
+		free(fdp->fd_ofiles, M_FILEDESC);
+	fdp->fd_ofiles = ntable;
+	fdp->fd_ofileflags = nfileflags;
+	if (NDSLOTS(nnfiles) > NDSLOTS(onfiles)) {
+		bcopy(fdp->fd_map, nmap, NDSLOTS(onfiles) * sizeof(*nmap));
+		if (NDSLOTS(onfiles) > NDSLOTS(NDFILE))
+			free(fdp->fd_map, M_FILEDESC);
+		fdp->fd_map = nmap;
+	}
+	fdp->fd_nfiles = nnfiles;
+}
+
+/*
+ * Allocate a file descriptor for the process.
+ */
+int
+fdalloc(struct thread *td, int *result)
+{
+	struct proc *p = td->td_proc;
+	struct filedesc *fdp = p->p_fd;
+	int fd = -1, maxfd;
+
+	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
+
+	maxfd = min((int)p->p_rlimit[RLIMIT_NOFILE].rlim_cur, maxfilesperproc);
+
+	/*
+	 * Search the bitmap for a free descriptor.  If none is found, try
+	 * to grow the file table.  Keep at it until we either get a file
+	 * descriptor or run into process or system limits; fdgrowtable()
+	 * may drop the filedesc lock, so we're in a race.
+	 */
+	for (;;) {
+		fd = fd_first_free(fdp, fdp->fd_freefile, fdp->fd_nfiles);
+		if (fd >= maxfd)
+			return (EMFILE);
+		if (fd < fdp->fd_nfiles)
+			break;
+		fdgrowtable(fdp, min(fdp->fd_nfiles * 2, maxfd));
+	}
+	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
+
+	/*
+	 * Perform some sanity checks, then mark the file descriptor as
+	 * used and return it to the caller.
+	 */
+	KASSERT(!fdisused(fdp, fd),
+	    ("fd_first_free() returned non-free descriptor"));
+	KASSERT(fdp->fd_ofiles[fd] == NULL,
+	    ("free descriptor isn't"));
+	fdp->fd_ofileflags[fd] = 0; /* XXX needed? */
+	fdused(fdp, fd);
+	fdp->fd_freefile = fd_first_free(fdp, fd, fdp->fd_nfiles);
+	*result = fd;
+	return (0);
 }

 /*
@ -1205,7 +1329,7 @@ falloc(td, resultfp, resultfd)
 		LIST_INSERT_HEAD(&filehead, fp, f_list);
 	}
 	sx_xunlock(&filelist_lock);
-	if ((error = fdalloc(td, 0, &i))) {
+	if ((error = fdalloc(td, &i))) {
 		FILEDESC_UNLOCK(p->p_fd);
 		fdrop(fp, td);
 		if (resultfp)
@ -1248,6 +1372,8 @@ fdinit(fdp)
 {
 	struct filedesc0 *newfdp;

+	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
+
 	MALLOC(newfdp, struct filedesc0 *, sizeof(struct filedesc0),
 	    M_FILEDESC, M_WAITOK | M_ZERO);
 	mtx_init(&newfdp->fd_fd.fd_mtx, FILEDESC_LOCK_DESC, NULL, MTX_DEF);
@ -1268,6 +1394,7 @@ fdinit(fdp)
 	newfdp->fd_fd.fd_ofileflags = newfdp->fd_dfileflags;
 	newfdp->fd_fd.fd_nfiles = NDFILE;
 	newfdp->fd_fd.fd_knlistsize = -1;
+	newfdp->fd_fd.fd_map = newfdp->fd_dmap;
 	return (&newfdp->fd_fd);
 }

@ -1294,107 +1421,38 @@ fdcopy(fdp)
 	struct filedesc *fdp;
 {
 	struct filedesc *newfdp;
-	struct file **fpp;
-	int i, j;
+	int i;

 	/* Certain daemons might not have file descriptors. */
 	if (fdp == NULL)
 		return (NULL);

 	FILEDESC_LOCK_ASSERT(fdp, MA_OWNED);
-
-	FILEDESC_UNLOCK(fdp);
-	MALLOC(newfdp, struct filedesc *, sizeof(struct filedesc0),
-	    M_FILEDESC, M_WAITOK);
-	FILEDESC_LOCK(fdp);
-	bcopy(fdp, newfdp, sizeof(struct filedesc));
-	FILEDESC_UNLOCK(fdp);
-	bzero(&newfdp->fd_mtx, sizeof(newfdp->fd_mtx));
-	mtx_init(&newfdp->fd_mtx, FILEDESC_LOCK_DESC, NULL, MTX_DEF);
-	if (newfdp->fd_cdir)
-		VREF(newfdp->fd_cdir);
-	if (newfdp->fd_rdir)
-		VREF(newfdp->fd_rdir);
-	if (newfdp->fd_jdir)
-		VREF(newfdp->fd_jdir);
-	newfdp->fd_refcnt = 1;
-
-	/*
-	 * If the number of open files fits in the internal arrays
-	 * of the open file structure, use them, otherwise allocate
-	 * additional memory for the number of descriptors currently
-	 * in use.
-	 */
-	FILEDESC_LOCK(fdp);
-	newfdp->fd_lastfile = fdp->fd_lastfile;
-	newfdp->fd_nfiles = fdp->fd_nfiles;
-	if (newfdp->fd_lastfile < NDFILE) {
-		newfdp->fd_ofiles = ((struct filedesc0 *) newfdp)->fd_dfiles;
-		newfdp->fd_ofileflags =
-		    ((struct filedesc0 *) newfdp)->fd_dfileflags;
-		i = NDFILE;
-	} else {
-		/*
-		 * Compute the smallest multiple of NDEXTENT needed
-		 * for the file descriptors currently in use,
-		 * allowing the table to shrink.
-		 */
-retry:
-		i = newfdp->fd_nfiles;
-		while (i > 2 * NDEXTENT && i > newfdp->fd_lastfile * 2)
-			i /= 2;
-		FILEDESC_UNLOCK(fdp);
-		MALLOC(newfdp->fd_ofiles, struct file **, i * OFILESIZE,
-		    M_FILEDESC, M_WAITOK);
-		FILEDESC_LOCK(fdp);
-		newfdp->fd_lastfile = fdp->fd_lastfile;
-		newfdp->fd_nfiles = fdp->fd_nfiles;
-		j = newfdp->fd_nfiles;
-		while (j > 2 * NDEXTENT && j > newfdp->fd_lastfile * 2)
-			j /= 2;
-		if (i != j) {
-			/*
-			 * The size of the original table has changed.
-			 * Go over once again.
-			 */
-			FILEDESC_UNLOCK(fdp);
-			FREE(newfdp->fd_ofiles, M_FILEDESC);
-			FILEDESC_LOCK(fdp);
-			newfdp->fd_lastfile = fdp->fd_lastfile;
-			newfdp->fd_nfiles = fdp->fd_nfiles;
-			goto retry;
+	newfdp = fdinit(fdp);
+	FILEDESC_LOCK(newfdp);
+	if (fdp->fd_lastfile >= newfdp->fd_nfiles)
+		fdgrowtable(newfdp, fdp->fd_lastfile + 1);
+	KASSERT(newfdp->fd_nfiles > fdp->fd_lastfile,
+	    ("fdgrowtable() failed to grow table"));
+	/* copy everything except kqueue descriptors */
+	newfdp->fd_freefile = -1;
+	for (i = 0; i <= fdp->fd_lastfile; ++i) {
+		if (fdisused(fdp, i) &&
+		    fdp->fd_ofiles[i]->f_type != DTYPE_KQUEUE) {
+			newfdp->fd_ofiles[i] = fdp->fd_ofiles[i];
+			newfdp->fd_ofileflags[i] = fdp->fd_ofileflags[i];
+			fdused(newfdp, i);
+			fhold(newfdp->fd_ofiles[i]);
+			newfdp->fd_lastfile = i;
+		} else {
+			if (newfdp->fd_freefile == -1)
+				newfdp->fd_freefile = i;
 		}
-		newfdp->fd_ofileflags = (char *) &newfdp->fd_ofiles[i];
-	}
-	newfdp->fd_nfiles = i;
-	bcopy(fdp->fd_ofiles, newfdp->fd_ofiles, i * sizeof(struct file **));
-	bcopy(fdp->fd_ofileflags, newfdp->fd_ofileflags, i * sizeof(char));
-
-	/*
-	 * kq descriptors cannot be copied.
-	 */
-	if (newfdp->fd_knlistsize != -1) {
-		fpp = &newfdp->fd_ofiles[newfdp->fd_lastfile];
-		for (i = newfdp->fd_lastfile; i >= 0; i--, fpp--) {
-			if (*fpp != NULL && (*fpp)->f_type == DTYPE_KQUEUE) {
-				*fpp = NULL;
-				if (i < newfdp->fd_freefile)
-					newfdp->fd_freefile = i;
-			}
-			if (*fpp == NULL && i == newfdp->fd_lastfile && i > 0)
-				newfdp->fd_lastfile--;
-		}
-		newfdp->fd_knlist = NULL;
-		newfdp->fd_knlistsize = -1;
-		newfdp->fd_knhash = NULL;
-		newfdp->fd_knhashmask = 0;
-	}
-
-	fpp = newfdp->fd_ofiles;
-	for (i = newfdp->fd_lastfile; i-- >= 0; fpp++) {
-		if (*fpp != NULL)
-			fhold(*fpp);
 	}
+	if (newfdp->fd_freefile == -1)
+		newfdp->fd_freefile = i;
+	newfdp->fd_cmask = fdp->fd_cmask;
+	FILEDESC_UNLOCK(newfdp);
 	return (newfdp);
 }

@ -1434,7 +1492,7 @@ fdfree(td)
 			i = 0;
 			fpp = fdp->fd_ofiles;
 			for (i = 0, fpp = fdp->fd_ofiles;
-			     i < fdp->fd_lastfile;
+			     i <= fdp->fd_lastfile;
 			     i++, fpp++) {
 				if (*fpp == NULL ||
 				    (*fpp)->f_type != DTYPE_VNODE)
@ -1518,6 +1576,8 @@ fdfree(td)

 	if (fdp->fd_nfiles > NDFILE)
 		FREE(fdp->fd_ofiles, M_FILEDESC);
+	if (NDSLOTS(fdp->fd_nfiles) > NDSLOTS(NDFILE))
+		FREE(fdp->fd_map, M_FILEDESC);
 	if (fdp->fd_cdir)
 		vrele(fdp->fd_cdir);
 	if (fdp->fd_rdir)
@ -1591,15 +1651,12 @@ setugidsafety(td)
 			fp = fdp->fd_ofiles[i];
 			fdp->fd_ofiles[i] = NULL;
 			fdp->fd_ofileflags[i] = 0;
-			if (i < fdp->fd_freefile)
-				fdp->fd_freefile = i;
+			fdunused(fdp, i);
 			FILEDESC_UNLOCK(fdp);
 			(void) closef(fp, td);
 			FILEDESC_LOCK(fdp);
 		}
 	}
-	while (fdp->fd_lastfile > 0 && fdp->fd_ofiles[fdp->fd_lastfile] == NULL)
-		fdp->fd_lastfile--;
 	FILEDESC_UNLOCK(fdp);
 }

@ -1641,15 +1698,12 @@ fdcloseexec(td)
 			fp = fdp->fd_ofiles[i];
 			fdp->fd_ofiles[i] = NULL;
 			fdp->fd_ofileflags[i] = 0;
-			if (i < fdp->fd_freefile)
-				fdp->fd_freefile = i;
+			fdunused(fdp, i);
 			FILEDESC_UNLOCK(fdp);
 			(void) closef(fp, td);
 			FILEDESC_LOCK(fdp);
 		}
 	}
-	while (fdp->fd_lastfile > 0 && fdp->fd_ofiles[fdp->fd_lastfile] == NULL)
-		fdp->fd_lastfile--;
 	FILEDESC_UNLOCK(fdp);
 }

@ -1698,6 +1752,7 @@ fdcheckstd(td)
 				FILEDESC_LOCK(fdp);
 				if (fdp->fd_ofiles[fd] == fp) {
 					fdp->fd_ofiles[fd] = NULL;
+					fdunused(fdp, fd);
 					extraref = 1;
 				}
 				FILEDESC_UNLOCK(fdp);
@ -2163,10 +2218,10 @@ dupfdopen(td, fdp, indx, dfd, mode, error)
 		fp = fdp->fd_ofiles[indx];
 		fdp->fd_ofiles[indx] = wfp;
 		fdp->fd_ofileflags[indx] = fdp->fd_ofileflags[dfd];
+		if (fp == NULL)
+			fdused(fdp, indx);
 		fhold_locked(wfp);
 		FILE_UNLOCK(wfp);
-		if (indx > fdp->fd_lastfile)
-			fdp->fd_lastfile = indx;
 		if (fp != NULL)
 			FILE_LOCK(fp);
 		FILEDESC_UNLOCK(fdp);
@ -2187,21 +2242,9 @@ dupfdopen(td, fdp, indx, dfd, mode, error)
 		fdp->fd_ofiles[dfd] = NULL;
 		fdp->fd_ofileflags[indx] = fdp->fd_ofileflags[dfd];
 		fdp->fd_ofileflags[dfd] = 0;
-
-		/*
-		 * Complete the clean up of the filedesc structure by
-		 * recomputing the various hints.
-		 */
-		if (indx > fdp->fd_lastfile) {
-			fdp->fd_lastfile = indx;
-		} else {
-			while (fdp->fd_lastfile > 0 &&
-			   fdp->fd_ofiles[fdp->fd_lastfile] == NULL) {
-				fdp->fd_lastfile--;
-			}
-			if (dfd < fdp->fd_freefile)
-				fdp->fd_freefile = dfd;
-		}
+		fdunused(fdp, dfd);
+		if (fp == NULL)
+			fdused(fdp, indx);
 		if (fp != NULL)
 			FILE_LOCK(fp);
 		FILEDESC_UNLOCK(fdp);
@ -2221,7 +2264,6 @@ dupfdopen(td, fdp, indx, dfd, mode, error)
 	/* NOTREACHED */
 }

-
 struct filedesc_to_leader *
 filedesc_to_leader_alloc(struct filedesc_to_leader *old,
 			 struct filedesc *fdp,
@ -2248,7 +2290,7 @@ filedesc_to_leader_alloc(struct filedesc_to_leader *old,
 		fdtol->fdl_next = fdtol;
 		fdtol->fdl_prev = fdtol;
 	}
-	return fdtol;
+	return (fdtol);
 }

 /*
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@ -258,6 +258,7 @@ pipe(td, uap)
 		FILEDESC_LOCK(fdp);
 		if (fdp->fd_ofiles[td->td_retval[0]] == rf) {
 			fdp->fd_ofiles[td->td_retval[0]] = NULL;
+			fdunused(fdp, td->td_retval[0]);
 			FILEDESC_UNLOCK(fdp);
 			fdrop(rf, td);
 		} else {
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@ -125,6 +125,7 @@ socket(td, uap)
 	if (error) {
 		if (fdp->fd_ofiles[fd] == fp) {
 			fdp->fd_ofiles[fd] = NULL;
+			fdunused(fdp, fd);
 			FILEDESC_UNLOCK(fdp);
 			fdrop(fp, td);
 		} else {
@ -391,6 +392,7 @@ accept1(td, uap, compat)
 		FILEDESC_LOCK(fdp);
 		if (fdp->fd_ofiles[fd] == nfp) {
 			fdp->fd_ofiles[fd] = NULL;
+			fdunused(fdp, fd);
 			FILEDESC_UNLOCK(fdp);
 			fdrop(nfp, td);
 		} else {
@ -585,6 +587,7 @@ socketpair(td, uap)
 	FILEDESC_LOCK(fdp);
 	if (fdp->fd_ofiles[sv[1]] == fp2) {
 		fdp->fd_ofiles[sv[1]] = NULL;
+		fdunused(fdp, sv[1]);
 		FILEDESC_UNLOCK(fdp);
 		fdrop(fp2, td);
 	} else {
@ -595,6 +598,7 @@ socketpair(td, uap)
 	FILEDESC_LOCK(fdp);
 	if (fdp->fd_ofiles[sv[0]] == fp1) {
 		fdp->fd_ofiles[sv[0]] = NULL;
+		fdunused(fdp, sv[0]);
 		FILEDESC_UNLOCK(fdp);
 		fdrop(fp1, td);
 	} else {
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@ -1047,7 +1047,7 @@ unp_externalize(control, controlp)
 			fdp = (int *)
 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
 			for (i = 0; i < newfds; i++) {
-				if (fdalloc(td, 0, &f))
+				if (fdalloc(td, &f))
 					panic("unp_externalize fdalloc failed");
 				fp = *rp++;
 				td->td_proc->p_fd->fd_ofiles[f] = fp;
--- a/sys/kern/vfs_extattr.c
+++ b/sys/kern/vfs_extattr.c
@ -998,6 +998,7 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 		FILEDESC_LOCK(fdp);
 		if (fdp->fd_ofiles[indx] == fp) {
 			fdp->fd_ofiles[indx] = NULL;
+			fdunused(fdp, indx);
 			FILEDESC_UNLOCK(fdp);
 			fdrop(fp, td);
 		} else {
@ -1093,6 +1094,7 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 	FILEDESC_LOCK(fdp);
 	if (fdp->fd_ofiles[indx] == fp) {
 		fdp->fd_ofiles[indx] = NULL;
+		fdunused(fdp, indx);
 		FILEDESC_UNLOCK(fdp);
 		fdrop(fp, td);
 	} else {
@ -3982,6 +3984,7 @@ fhopen(td, uap)
 			FILEDESC_LOCK(fdp);
 			if (fdp->fd_ofiles[indx] == fp) {
 				fdp->fd_ofiles[indx] = NULL;
+				fdunused(fdp, indx);
 				FILEDESC_UNLOCK(fdp);
 				fdrop(fp, td);
 			} else {
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@ -998,6 +998,7 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 		FILEDESC_LOCK(fdp);
 		if (fdp->fd_ofiles[indx] == fp) {
 			fdp->fd_ofiles[indx] = NULL;
+			fdunused(fdp, indx);
 			FILEDESC_UNLOCK(fdp);
 			fdrop(fp, td);
 		} else {
@ -1093,6 +1094,7 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 	FILEDESC_LOCK(fdp);
 	if (fdp->fd_ofiles[indx] == fp) {
 		fdp->fd_ofiles[indx] = NULL;
+		fdunused(fdp, indx);
 		FILEDESC_UNLOCK(fdp);
 		fdrop(fp, td);
 	} else {
@ -3982,6 +3984,7 @@ fhopen(td, uap)
 			FILEDESC_LOCK(fdp);
 			if (fdp->fd_ofiles[indx] == fp) {
 				fdp->fd_ofiles[indx] = NULL;
+				fdunused(fdp, indx);
 				FILEDESC_UNLOCK(fdp);
 				fdrop(fp, td);
 			} else {
--- a/sys/sys/filedesc.h
+++ b/sys/sys/filedesc.h
@ -39,6 +39,7 @@

 #include <sys/_lock.h>
 #include <sys/_mutex.h>
+#include <sys/limits.h> /* XXX for CHAR_BIT */
 #include <sys/queue.h>

 /*
@ -50,13 +51,15 @@
 * the historical limit of 20 open files (and the usage of descriptors by
 * shells).  If these descriptors are exhausted, a larger descriptor table
 * may be allocated, up to a process' resource limit; the internal arrays
- * are then unused.  The initial expansion is set to NDEXTENT; each time
- * it runs out, it is doubled until the resource limit is reached. NDEXTENT
- * should be selected to be the biggest multiple of OFILESIZE (see below)
- * that will fit in a power-of-two sized piece of memory.
+ * are then unused.
 */
 #define NDFILE		20
-#define NDEXTENT	50		/* 250 bytes in 256-byte alloc. */
+#define NDSLOTTYPE	unsigned long
+#define NDSLOTSIZE	sizeof(NDSLOTTYPE)
+#define	NDENTRIES	(NDSLOTSIZE * CHAR_BIT)
+#define NDSLOT(x)	((x) / NDENTRIES)
+#define NDBIT(x)	((NDSLOTTYPE)1 << ((x) % NDENTRIES))
+#define	NDSLOTS(x)	(((x) + NDENTRIES - 1) / NDENTRIES)

 struct filedesc {
 	struct	file **fd_ofiles;	/* file structures for open files */
@ -65,6 +68,7 @@ struct filedesc {
 	struct	vnode *fd_rdir;		/* root directory */
 	struct	vnode *fd_jdir;		/* jail root directory */
 	int	fd_nfiles;		/* number of open files allocated */
+	NDSLOTTYPE	*fd_map;	/* bitmap of free fds */
 	int	fd_lastfile;		/* high-water mark of fd_ofiles */
 	int	fd_freefile;		/* approx. next free file */
 	u_short	fd_cmask;		/* mask for file creation */
@ -91,6 +95,7 @@ struct filedesc0 {
 	 */
 	struct	file *fd_dfiles[NDFILE];
 	char	fd_dfileflags[NDFILE];
+	NDSLOTTYPE	fd_dmap[NDSLOTS(NDFILE)];
 };


@ -140,7 +145,9 @@ int	closef(struct file *fp, struct thread *p);
 int	dupfdopen(struct thread *td, struct filedesc *fdp, int indx, int dfd,
 	    int mode, int error);
 int	falloc(struct thread *p, struct file **resultfp, int *resultfd);
-int	fdalloc(struct thread *p, int want, int *result);
+void	fdused(struct filedesc *fdp, int fd);
+void	fdunused(struct filedesc *fdp, int fd);
+int	fdalloc(struct thread *p, int *result);
 int	fdavail(struct thread *td, int n);
 void	fdcloseexec(struct thread *td);
 int	fdcheckstd(struct thread *td);