sh: Do not try to execute binary files as scripts.

If execve() returns an [ENOEXEC] error, check if the file is binary before
trying to execute it using sh. A file is considered binary if at least one
of the first 256 bytes is '\0'.

In particular, trying to execute ELF binaries for the wrong architecture now
fails with an "Exec format error" message instead of syntax errors and
potentially strange results.

bin/sh/exec.c

@@ -126,6 +126,8 @@ shellexec(char **argv, char **envp, const char *path, int idx)
 				tryexec(cmdname, argv, envp);
 				if (errno != ENOENT && errno != ENOTDIR)
 					e = errno;
+				if (e == ENOEXEC)
+					break;
 			}
 			stunalloc(cmdname);
 		}
@@ -145,11 +147,23 @@ shellexec(char **argv, char **envp, const char *path, int idx)
 static void
 tryexec(char *cmd, char **argv, char **envp)
 {
-	int e;
+	int e, in;
+	ssize_t n;
+	char buf[256];
 
 	execve(cmd, argv, envp);
 	e = errno;
 	if (e == ENOEXEC) {
+		INTOFF;
+		in = open(cmd, O_RDONLY | O_NONBLOCK);
+		if (in != -1) {
+			n = pread(in, buf, sizeof buf, 0);
+			close(in);
+			if (n > 0 && memchr(buf, '\0', n) != NULL) {
+				errno = ENOEXEC;
+				return;
+			}
+		}
 		*argv = cmd;
 		*--argv = _PATH_BSHELL;
 		execve(_PATH_BSHELL, argv, envp);

bin/sh/sh.1

@@ -647,6 +647,7 @@ resulting in an
 .Er ENOEXEC
 return value from
 .Xr execve 2 )
+but appears to be a text file,
 the shell will run a new instance of
 .Nm
 to interpret it.

tools/regression/bin/sh/errors/bad-binary1.126

@@ -0,0 +1,12 @@
+# $FreeBSD$
+# Checking for binary "scripts" without magic number is permitted but not
+# required by POSIX. However, it is preferable to getting errors like
+# Syntax error: word unexpected (expecting ")")
+# from trying to execute ELF binaries for the wrong architecture.
+
+T=`mktemp -d "${TMPDIR:-/tmp}/sh-test.XXXXXXXX"` || exit
+trap 'rm -rf "${T}"' 0
+printf '\0echo bad\n' >"$T/testshellproc"
+chmod 755 "$T/testshellproc"
+PATH=$T:$PATH
+testshellproc 2>/dev/null