Major optimizations for 'make index' and other recursive traversal

targets. * Use /rescue/sh for index builds instead of /bin/sh, when it exists. The former is statically linked and faster to execute, which becomes significant when executing it tens of thousands of times. This trick can be used with other recursive targets by passing in __MAKE_SHELL. * Get rid of make variable assignments that use != command invocations in the critical path, using several methods: - rewriting logic to use shell or make builtins instead of external command executions - macroizing commands and executing them in the targets where they are needed instead of with every invocation of make - precomputing the results of invariant commands in bsd.port.subdir.mk and passing them in explicitly to child makes, and using this to avoid recalculation in all the children. NB: the commands are still run one per top-level subdirectory but this does not currently seem to be a major issue. They could be moved further up into the top-level Makefile at the cost of some cleanliness. - Committers are strongly discouraged from adding further "bare" != assignments to the ports tree, even in their own ports. One of the above strategies should be used to avoid future bloat. * Rewrite the core 'describe' target to work entirely within a single shell process using only builtin commands. The old version is retained as a backup for use on systems older than 603104, which does not have the make :u modifier. This cuts down the number of processes executed during the course of a 'make index' by an order of magnitude, and we are essentially now amortized to the minimum of a single make + sh instance per port, plus whatever commands the port makefile itself executes (which are usually unnecessary and bogus). * Less validation of the WWW: target is performed; this can become policed at a port level by portlint. Specifically we look at the second word of the first line beginning with "WWW:" in pkg-descr, and append "http://" to it unless it already begins with "http://", "https://" or "ftp://". Thanks to dougb for the idea of how to extract WWW: using shell builtins. * Use the "true" shell builtin instead of echo > /dev/null for a measurable decrease in CPU use. * Add a note about dubious escaping strategy in bsd.port.subdir.mk * Minor change in output of 'make describe': it no longer strips trailing CR characters from pkg-descr files with MSDOS CR/LF termination. Instead the makeindex perl script that post-processes make describe into the INDEX is tweaked to strip on input. The bottom line is that on my test hardware INDEX builds are now faster by more than a factor of 2 and with a reduction in system time by a factor of 4-8 depending on configuration.
svn path=/head/; revision=217132
2024-12-20 04:02:27 +00:00 · 2008-07-19 17:59:41 +00:00 · 2008-07-19 17:59:41 +00:00 · 54e565eedc · 2021-03-31 03:12:20 +00:00
commit 54e565eedc
parent a8c31a5d50
6 changed files with 211 additions and 73 deletions
--- a/12
+++ b/12
@ -86,13 +86,22 @@ FETCHINDEX?=	${SETENV} ${FETCH_ENV} fetch -am -o
 INDEX_JOBS?=	2

 .if !defined(INDEX_VERBOSE)
-INDEX_ECHO_MSG=		echo > /dev/null
+INDEX_ECHO_MSG=		true
 INDEX_ECHO_1ST=		echo -n
 .else
 INDEX_ECHO_MSG=		echo 1>&2
 INDEX_ECHO_1ST=		echo
 .endif

+# /rescue/sh is statically linked and much faster to execute than the
+# dynamically linked /bin/sh.  This is significant for targets like
+# make index that execute the shell tens of thousands of times.
+.if exists(/rescue/sh)
+INDEX_SHELL=		/rescue/sh
+.else
+INDEX_SHELL=		/bin/sh
+.endif
+
 ${INDEXDIR}/${INDEXFILE}:
 	@${INDEX_ECHO_1ST} "Generating ${INDEXFILE} - please wait.."; \
 	if [ "${INDEX_PRISTINE}" != "" ]; then \
@ -103,6 +112,7 @@ ${INDEXDIR}/${INDEXFILE}:
 	tmpdir=`/usr/bin/mktemp -d -t index` || exit 1; \
 	trap "rm -rf $${tmpdir}; exit 1" 1 2 3 5 10 13 15; \
 	( cd ${.CURDIR} && make -j${INDEX_JOBS} INDEX_TMPDIR=$${tmpdir} BUILDING_INDEX=1 \
+		__MAKE_SHELL=${INDEX_SHELL} \
 		ECHO_MSG="${INDEX_ECHO_MSG}" describe ) || \
 		(rm -rf $${tmpdir} ; \
 		if [ "${INDEX_QUIET}" = "" ]; then \
--- a/Mk/bsd.java.mk
+++ b/Mk/bsd.java.mk
@ -249,34 +249,30 @@ check-makevars::
 .		endfor

 # Error checking: JAVA_VERSION
+.if !defined(_JAVA_VERSION_LIST_REGEXP)
 _JAVA_VERSION_LIST_REGEXP!=		${ECHO_CMD} "${_JAVA_VERSION_LIST}" | ${SED} "s/ /\\\|/g"
-_ERROR_CHECKING_JAVA_VERSION!=	${ECHO_CMD} "${JAVA_VERSION}" | ${TR} " " "\n" \
-								| ${GREP} -v "${_JAVA_VERSION_LIST_REGEXP}" || true
-.		if (${_ERROR_CHECKING_JAVA_VERSION} != "")
+.endif
 check-makevars::
-	@${ECHO_CMD} "${PKGNAME}: Makefile error: \"${JAVA_VERSION}\" is not a valid value for JAVA_VERSION. It should be one or more of: ${__JAVA_VERSION_LIST} (with an optional \"+\" suffix.)";
-	@${FALSE}
-.		endif
+	@test ! -z "${JAVA_VERSION}" && ( ${ECHO_CMD} "${JAVA_VERSION}" | ${TR} " " "\n" | ${GREP} -q "${_JAVA_VERSION_LIST_REGEXP}" || \
+	(${ECHO_CMD} "${PKGNAME}: Makefile error: \"${JAVA_VERSION}\" is not a valid value for JAVA_VERSION. It should be one or more of: ${__JAVA_VERSION_LIST} (with an optional \"+\" suffix.)"; ${FALSE})) || true

 # Error checking: JAVA_VENDOR
+.if !defined(_JAVA_VENDOR_LIST_REGEXP)
 _JAVA_VENDOR_LIST_REGEXP!=		${ECHO_CMD} "${_JAVA_VENDOR_LIST}" | ${SED} "s/ /\\\|/g"
-_ERROR_CHECKING_JAVA_VENDOR!=	${ECHO_CMD} "${JAVA_VENDOR}" | ${TR} " " "\n" \
-								| ${GREP} -v "${_JAVA_VENDOR_LIST_REGEXP}" || true
-.		if (${_ERROR_CHECKING_JAVA_VENDOR} != "")
+.endif
 check-makevars::
-	@${ECHO_CMD} "${PKGNAME}: Makefile error: \"${JAVA_VENDOR}\" is not a valid value for JAVA_VENDOR. It should be one or more of: ${_JAVA_VENDOR_LIST}";
-	@${FALSE}
-.		endif
+	@test ! -z "${JAVA_VENDOR}" && ( ${ECHO_CMD} "${JAVA_VENDOR}" | ${TR} " " "\n" | ${GREP} -q "${_JAVA_VENDOR_LIST_REGEXP}" || \
+	(${ECHO_CMD} "${PKGNAME}: Makefile error: \"${JAVA_VENDOR}\" is not a valid value for JAVA_VENDOR. It should be one or more of: ${_JAVA_VENDOR_LIST}"; \
+	${FALSE})) || true

 # Error checking: JAVA_OS
+.if !defined(_JAVA_OS_LIST_REGEXP)
 _JAVA_OS_LIST_REGEXP!=		${ECHO_CMD} "${_JAVA_OS_LIST}" | ${SED} "s/ /\\\|/g"
-_ERROR_CHECKING_JAVA_OS!=	${ECHO_CMD} "${JAVA_OS}" | ${TR} " " "\n" \
-							| ${GREP} -v "${_JAVA_OS_LIST_REGEXP}" || true
-.		if (${_ERROR_CHECKING_JAVA_OS} != "")
+.endif
 check-makevars::
-	@${ECHO_CMD} "${PKGNAME}: Makefile error: \"${JAVA_OS}\" is not a valid value for JAVA_OS. It should be one or more of: ${_JAVA_OS_LIST}";
-	@${FALSE}
-.		endif
+	@test ! -z "${JAVA_OS}" && ( ${ECHO_CMD} "${JAVA_OS}" | ${TR} " " "\n" | ${GREP} -q "${_JAVA_OS_LIST_REGEXP}" || \
+	(${ECHO_CMD} "${PKGNAME}: Makefile error: \"${JAVA_OS}\" is not a valid value for JAVA_OS. It should be one or more of: ${_JAVA_OS_LIST}"; \
+	${FALSE})) || true

 # Set default values for JAVA_BUILD and JAVA_RUN
 # When nothing is set, assume JAVA_BUILD=jdk and JAVA_RUN=jre
@ -313,18 +309,30 @@ A_JAVA_PORT_HOME=			${A_JAVA_PORT_INFO:MHOME=*:S,HOME=,,}
 A_JAVA_PORT_VERSION=		${A_JAVA_PORT_INFO:MVERSION=*:C/VERSION=([0-9])\.([0-9])(.*)/\1.\2/}
 A_JAVA_PORT_OS=				${A_JAVA_PORT_INFO:MOS=*:S,OS=,,}
 A_JAVA_PORT_VENDOR=			${A_JAVA_PORT_INFO:MVENDOR=*:S,VENDOR=,,}
+.if !defined(_JAVA_PORTS_INSTALLED)
 A_JAVA_PORT_INSTALLED!=		${TEST} -x "${A_JAVA_PORT_HOME}/${_JDK_FILE}" \
 							&& ${ECHO_CMD} "${A_JAVA_PORT}" \
 							|| ${TRUE}
 __JAVA_PORTS_INSTALLED!=	${ECHO_CMD} "${__JAVA_PORTS_INSTALLED} ${A_JAVA_PORT_INSTALLED}"
-A_JAVA_PORT_POSSIBLE!=		${ECHO_CMD} "${_JAVA_VERSION}" | ${GREP} -q "${A_JAVA_PORT_VERSION}" \
-							&& ${ECHO_CMD} "${_JAVA_OS}" | ${GREP} -q "${A_JAVA_PORT_OS}" \
-							&& ${ECHO_CMD} "${_JAVA_VENDOR}" | ${GREP} -q "${A_JAVA_PORT_VENDOR}" \
-							&& ${ECHO_CMD} "${A_JAVA_PORT}" \
-							|| ${TRUE}
-__JAVA_PORTS_POSSIBLE!=		${ECHO_CMD} "${__JAVA_PORTS_POSSIBLE} ${A_JAVA_PORT_POSSIBLE}"
+.endif
+
+# The magic here is that we want to test for a substring using only shell builtins (to avoid forking)
+# Our shell does not have an explicit substring operator, but we can build one by using the '#'
+# deletion operator ('%' would also work).  We try to delete the pattern "*${substr}*" and compare it
+# to the original string.  If they differ, the substring matched.
+# 
+# We can't do this in make because it doesn't allow nested modifiers ${foo:${bar}}
+#
+A_JAVA_PORT_POSSIBLE!=		ver="${_JAVA_VERSION}"; os="${_JAVA_OS}"; vendor="${_JAVA_VENDOR}"; \
+				${TEST} "$${ver\#*${A_JAVA_PORT_VERSION}*}" != "${_JAVA_VERSION}" -a \
+				"$${os\#*${A_JAVA_PORT_OS}*}" != "${_JAVA_OS}" -a \
+				"$${vendor\#*${A_JAVA_PORT_VENDOR}*}" != "${_JAVA_VENDOR}" && \
+				${ECHO_CMD} "${A_JAVA_PORT}" || ${TRUE}
+__JAVA_PORTS_POSSIBLE:=		${__JAVA_PORTS_POSSIBLE} ${A_JAVA_PORT_POSSIBLE}
 .		endfor
+.if !defined(_JAVA_PORTS_INSTALLED)
 _JAVA_PORTS_INSTALLED=		${__JAVA_PORTS_INSTALLED:C/ [ ]+/ /g}
+.endif
 _JAVA_PORTS_POSSIBLE=		${__JAVA_PORTS_POSSIBLE:C/ [ ]+/ /g}


@ -337,20 +345,28 @@ _JAVA_PORTS_POSSIBLE=		${__JAVA_PORTS_POSSIBLE:C/ [ ]+/ /g}
 .		undef _JAVA_PORTS_INSTALLED_POSSIBLE

 .		for A_JAVA_PORT in ${_JAVA_PORTS_POSSIBLE}
-A_JAVA_PORT_INSTALLED_POSSIBLE!=	${ECHO_CMD} "${_JAVA_PORTS_INSTALLED}" | ${GREP} -q "${A_JAVA_PORT}" \
-									&& ${ECHO_CMD} "${A_JAVA_PORT}" || ${TRUE}
-__JAVA_PORTS_INSTALLED_POSSIBLE!=	${ECHO_CMD} "${__JAVA_PORTS_INSTALLED_POSSIBLE} ${A_JAVA_PORT_INSTALLED_POSSIBLE}"
+A_JAVA_PORT_INSTALLED_POSSIBLE!=	inst="${_JAVA_PORTS_INSTALLED}"; \
+					${TEST} "$${inst\#*${A_JAVA_PORT}*}" != "${_JAVA_PORTS_INSTALLED}" && \
+					${ECHO_CMD} "${A_JAVA_PORT}" || ${TRUE}
+__JAVA_PORTS_INSTALLED_POSSIBLE:=	${__JAVA_PORTS_INSTALLED_POSSIBLE} ${A_JAVA_PORT_INSTALLED_POSSIBLE}
 .		endfor
-_JAVA_PORTS_INSTALLED_POSSIBLE=		${__JAVA_PORTS_INSTALLED_POSSIBLE:C/ [ ]+/ /g}
+_JAVA_PORTS_INSTALLED_POSSIBLE=		${__JAVA_PORTS_INSTALLED_POSSIBLE:C/[ ]+//g}

 .		if ${_JAVA_PORTS_INSTALLED_POSSIBLE} != ""
-_JAVA_PORT!=	${ECHO_CMD} "${_JAVA_PORTS_INSTALLED_POSSIBLE}" \
-				| ${AWK} '{ print $$1 }'
-
+.                 for i in ${_JAVA_PORTS_INSTALLED_POSSIBLE}
+.                   if !defined(_JAVA_PORTS_INSTALLED_POSSIBLE_shortcircuit)
+_JAVA_PORT=	$i
+_JAVA_PORTS_INSTALLED_POSSIBLE_shortcircuit=	1
+.                   endif
+.                 endfor
 # If no installed JDK port fits, then pick one from the list of possible ones
 .		else
-_JAVA_PORT!=	${ECHO_CMD} "${_JAVA_PORTS_POSSIBLE}" \
-				| ${AWK} '{ print $$1 }'
+.                 for i in ${_JAVA_PORTS_POSSIBLE}
+.                   if !defined(_JAVA_PORTS_POSSIBLE_shortcircuit)
+_JAVA_PORT=	$i
+_JAVA_PORTS_POSSIBLE_shortcircuit=	1
+.                   endif
+.                 endfor
 .		endif

 _JAVA_PORT_INFO:=		${_JAVA_PORT:S/^/\${_/:S/$/_INFO}/}
--- a/Mk/bsd.port.mk
+++ b/Mk/bsd.port.mk
@ -1241,14 +1241,13 @@ UNIQUENAME?=	${LATEST_LINK}
 UNIQUENAME?=	${PKGNAMEPREFIX}${PORTNAME}
 .endif
 OPTIONSFILE?=	${PORT_DBDIR}/${UNIQUENAME}/options
-_OPTIONSFILE!=	${ECHO_CMD} "${OPTIONSFILE}"
 .if defined(OPTIONS)
 # include OPTIONSFILE first if exists
-.	if exists(${_OPTIONSFILE}) && !make(rmconfig)
-.	include "${_OPTIONSFILE}"
+.	if exists(${OPTIONSFILE}) && !make(rmconfig)
+.	include "${OPTIONSFILE}"
 .	endif
-.	if exists(${_OPTIONSFILE}.local)
-.	include "${_OPTIONSFILE}.local"
+.	if exists(${OPTIONSFILE}.local)
+.	include "${OPTIONSFILE}.local"
 .	endif
 WITHOUT:=
 WITH:=
@ -1654,7 +1653,7 @@ PATCH_DEPENDS+=		unzip:${PORTSDIR}/archivers/unzip
 HAVE_COMPAT_IA32_LIBS?=  YES
 .endif
 .if !defined(HAVE_COMPAT_IA32_KERN)
-HAVE_COMPAT_IA32_KERN!= if ${SYSCTL} -a compat.ia32.maxvmem >/dev/null 2>&1; then echo YES; fi
+HAVE_COMPAT_IA32_KERN!= if ${SYSCTL} -n compat.ia32.maxvmem >/dev/null 2>&1; then echo YES; fi
 .endif
 .endif

@ -3283,8 +3282,8 @@ options-message:
 .endif
 .if defined(_OPTIONS_READ)
 	@${ECHO_MSG} "===>  Found saved configuration for ${_OPTIONS_READ}"
-.if ${OPTIONSFILE} != ${_OPTIONSFILE}
-	@${ECHO_MSG} "===>  *** CAUTION *** Using wrong configuration file ${_OPTIONSFILE}"
+.if ${OPTIONSFILE} != ${OPTIONSFILE}
+	@${ECHO_MSG} "===>  *** CAUTION *** Using wrong configuration file ${OPTIONSFILE}"
 .endif
 .endif

@ -5300,14 +5299,46 @@ missing:
 # If this ever changes, portmgr should contact the portsnap maintainer
 # first to avoid gratuitous breakage.

-.if !target(describe)
+# XXX Older versions do not support the :u make modifier.  The .else
+# clause can be removed once 6.3-RELEASE is no longer supported.
+.if ${OSVERSION} >= 603104
+. if !target(describe)
+_EXTRACT_DEPENDS=${EXTRACT_DEPENDS:C/^[^ :]+:([^ :]+)(:[^ :]+)?/\1/:O:u}
+_PATCH_DEPENDS=${PATCH_DEPENDS:C/^[^ :]+:([^ :]+)(:[^ :]+)?/\1/:O:u}
+_FETCH_DEPENDS=${FETCH_DEPENDS:C/^[^ :]+:([^ :]+)(:[^ :]+)?/\1/:O:u} 
+_LIB_DEPENDS=${LIB_DEPENDS:C/^[^ :]+:([^ :]+)(:[^ :]+)?/\1/:O:u}
+_BUILD_DEPENDS=${BUILD_DEPENDS:C/^[^ :]+:([^ :]+)(:[^ :]+)?/\1/:O:u} ${_LIB_DEPENDS}
+_RUN_DEPENDS=${RUN_DEPENDS:C/^[^ :]+:([^ :]+)(:[^ :]+)?/\1/:O:u} ${_LIB_DEPENDS}
+. if exists(${DESCR})
+_DESCR=${DESCR}
+. else
+_DESCR=/dev/null
+. endif
+
+describe:
+	@${ECHO_CMD} -n "${PKGNAME}|${.CURDIR}|${PREFIX}|"; \
+	${ECHO_CMD} -n ${COMMENT:Q}; \
+	${ECHO_CMD} -n "|${_DESCR}|${MAINTAINER}|${CATEGORIES}|${_EXTRACT_DEPENDS}|${_PATCH_DEPENDS}|${_FETCH_DEPENDS}|${_BUILD_DEPENDS:O:u}|${_RUN_DEPENDS:O:u}|"; \
+	while read one two discard; do \
+		case "$$one" in \
+		WWW:)   case "$$two" in \
+			https://*|http://*|ftp://*) ${ECHO_CMD} -n "$$two" ;; \
+			*) ${ECHO_CMD} -n "http://$$two" ;; \
+			esac; \
+			break; \
+			;; \
+		esac; \
+	done < ${DESCR}; ${ECHO_CMD} 
+. endif
+.else
+. if !target(describe)
 describe:
 	@${ECHO_CMD} -n "${PKGNAME}|${.CURDIR}|${PREFIX}|"
-.if defined(COMMENT)
+. if defined(COMMENT)
 	@${ECHO_CMD} -n ${COMMENT:Q}
-.else
+. else
 	@${ECHO_CMD} -n '** No Description'
-.endif
+. endif
 	@perl -e ' \
 		if ( -f q{${DESCR}} ) { \
 			print q{|${DESCR}}; \
@ -5362,6 +5393,7 @@ describe:
 			} \
 		} \
 		print qq{\n};'
+. endif
 .endif

 www-site:
@ -5786,23 +5818,23 @@ config:
 .if !defined(OPTIONS)
 	@${ECHO_MSG} "===> No options to configure"
 .else
-.if ${OPTIONSFILE} != ${_OPTIONSFILE}
-	@${ECHO_MSG} "===> Using wrong configuration file ${_OPTIONSFILE}"
+.if ${OPTIONSFILE} != ${OPTIONSFILE}
+	@${ECHO_MSG} "===> Using wrong configuration file ${OPTIONSFILE}"
 	@exit 1
 .endif
 .if ${UID} != 0 && !defined(INSTALL_AS_USER)
-	@optionsdir=${_OPTIONSFILE}; optionsdir=$${optionsdir%/*}; \
+	@optionsdir=${OPTIONSFILE}; optionsdir=$${optionsdir%/*}; \
 	${ECHO_MSG} "===>  Switching to root credentials to create $${optionsdir}"; \
 	(${SU_CMD} "${SH} -c \"${MKDIR} $${optionsdir} 2> /dev/null\"") || \
 		(${ECHO_MSG} "===> Cannot create $${optionsdir}, check permissions"; exit 1); \
 	${ECHO_MSG} "===>  Returning to user credentials"
 .else
-	@(optionsdir=${_OPTIONSFILE}; optionsdir=$${optionsdir%/*}; \
+	@(optionsdir=${OPTIONSFILE}; optionsdir=$${optionsdir%/*}; \
 	${MKDIR} $${optionsdir} 2> /dev/null) || \
 		(${ECHO_MSG} "===> Cannot create $${optionsdir}, check permissions"; exit 1)
 .endif
-	-@if [ -e ${_OPTIONSFILE} ]; then \
-		. ${_OPTIONSFILE}; \
+	-@if [ -e ${OPTIONSFILE} ]; then \
+		. ${OPTIONSFILE}; \
 	fi; \
 	set -- ${OPTIONS} XXX; \
 	while [ $$# -gt 3 ]; do \
@ -5852,11 +5884,11 @@ config:
 		fi; \
 	done; \
 	if [ `${ID} -u` != 0 -a "x${INSTALL_AS_USER}" = "x" ]; then \
-		${ECHO_MSG} "===>  Switching to root credentials to write ${_OPTIONSFILE}"; \
-		${SU_CMD} "${CAT} $${TMPOPTIONSFILE} > ${_OPTIONSFILE}"; \
+		${ECHO_MSG} "===>  Switching to root credentials to write ${OPTIONSFILE}"; \
+		${SU_CMD} "${CAT} $${TMPOPTIONSFILE} > ${OPTIONSFILE}"; \
 		${ECHO_MSG} "===>  Returning to user credentials"; \
 	else \
-		${CAT} $${TMPOPTIONSFILE} > ${_OPTIONSFILE}; \
+		${CAT} $${TMPOPTIONSFILE} > ${OPTIONSFILE}; \
 	fi; \
 	${RM} -f $${TMPOPTIONSFILE}
 .endif
@ -5873,9 +5905,9 @@ config-recursive:
 .if !target(config-conditional)
 config-conditional:
 .if defined(OPTIONS)
-.if exists(${_OPTIONSFILE})
+.if exists(${OPTIONSFILE})
 # scan saved options and invalidate them, if the set of options does not match
-	@. ${_OPTIONSFILE}; \
+	@. ${OPTIONSFILE}; \
 	set ${OPTIONS} XXX; \
 	while [ $$# -gt 3 ]; do \
 		withvar=WITH_$$1; \
@ -5907,8 +5939,8 @@ config-conditional:
 showconfig:
 .if defined(OPTIONS)
 	@${ECHO_MSG} "===> The following configuration options are available for ${PKGNAME}:"
-	-@if [ -e ${_OPTIONSFILE} ]; then \
-		. ${_OPTIONSFILE}; \
+	-@if [ -e ${OPTIONSFILE} ]; then \
+		. ${OPTIONSFILE}; \
 	fi; \
 	set -- ${OPTIONS} XXX; \
 	while [ $$# -gt 3 ]; do \
@ -5933,16 +5965,16 @@ showconfig:

 .if !target(rmconfig)
 rmconfig:
-.if defined(OPTIONS) && exists(${_OPTIONSFILE})
+.if defined(OPTIONS) && exists(${OPTIONSFILE})
 	-@${ECHO_MSG} "===> Removing user-configured options for ${PKGNAME}"; \
-	optionsdir=${_OPTIONSFILE}; optionsdir=$${optionsdir%/*}; \
+	optionsdir=${OPTIONSFILE}; optionsdir=$${optionsdir%/*}; \
 	if [ `${ID} -u` != 0 -a "x${INSTALL_AS_USER}" = "x" ]; then \
-		${ECHO_MSG} "===> Switching to root credentials to remove ${_OPTIONSFILE} and $${optionsdir}"; \
-		${SU_CMD} "${RM} -f ${_OPTIONSFILE} ; \
+		${ECHO_MSG} "===> Switching to root credentials to remove ${OPTIONSFILE} and $${optionsdir}"; \
+		${SU_CMD} "${RM} -f ${OPTIONSFILE} ; \
 			${RMDIR} $${optionsdir}"; \
 		${ECHO_MSG} "===> Returning to user credentials"; \
 	else \
-		${RM} -f ${_OPTIONSFILE}; \
+		${RM} -f ${OPTIONSFILE}; \
 		${RMDIR} $${optionsdir}; \
 	fi
 .else
--- a/Mk/bsd.port.subdir.mk
+++ b/Mk/bsd.port.subdir.mk
@ -63,13 +63,15 @@ DESCR?=			${PKGDIR}/pkg-descr
 STRIP?=	-s
 .endif

+# These are variables that are invariant for the lifetime of a recursive port traversal
+# (index build, etc), so it is more efficient to precompute them here and pass them in
+# to child makes explicitly, instead of recomputing them tens of thousands of times.
+
 .if !defined(NOPRECIOUSMAKEVARS)
 .if !defined(ARCH)
 ARCH!=	${UNAME} -p
 .endif
-.if !defined(OSREL)
-OSREL!=	${UNAME} -r | ${SED} -e 's/[-(].*//'
-.endif
+
 .if !defined(OSVERSION)
 .if exists(/usr/include/sys/param.h)
 OSVERSION!=	${AWK} '/^\#define[[:blank:]]__FreeBSD_version/ {print $$3}' < /usr/include/sys/param.h
@ -79,23 +81,76 @@ OSVERSION!=	${AWK} '/^\#define[[:blank:]]__FreeBSD_version/ {print $$3}' < /usr/
 OSVERSION!=	${SYSCTL} -n kern.osreldate
 .endif
 .endif
+
+.if !defined(_OSRELEASE)
+_OSRELEASE!=			uname -r
+.endif
+.if !defined(OSREL)
+OSREL=	${_OSRELEASE:C/[-(].*//}
 .endif

-INDEXDIR?=	${PORTSDIR}
-INDEXFILE?=	INDEX-${OSVERSION:C/([0-9]).*/\1/}
+.if !defined(OPSYS)
+OPSYS!=	${UNAME} -s
+.endif

+.if ${ARCH} == "amd64" || ${ARCH} =="ia64"
+.if !defined(HAVE_COMPAT_IA32_KERN)
+HAVE_COMPAT_IA32_KERN!= if ${SYSCTL} -n compat.ia32.maxvmem >/dev/null 2>&1; then echo YES; fi
+.endif
+.endif
+
+.if !defined(CONFIGURE_MAX_CMD_LEN)
+CONFIGURE_MAX_CMD_LEN!= ${SYSCTL} -n kern.argmax
+.endif
+
+.if !defined(PYTHON_DEFAULT_VERSION)
+PYTHON_DEFAULT_VERSION!=	make -V PYTHON_DEFAULT_VERSION USE_PYTHON=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(PYTHON_DEFAULT_PORTVERSION)
+# We are caching the PYTHON_PORTVERSION of the default python version so we can reuse it in the
+# common case.
+PYTHON_DEFAULT_PORTVERSION!=	make -V PYTHON_PORTVERSION USE_PYTHON=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(PYTHONBASE)
+PYTHONBASE!=			make -V PYTHONBASE USE_PYTHON=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(_JAVA_VERSION_LIST_REGEXP)
+_JAVA_VERSION_LIST_REGEXP!=	make -V _JAVA_VERSION_LIST_REGEXP USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(_JAVA_VENDOR_LIST_REGEXP)
+_JAVA_VENDOR_LIST_REGEXP!=	make -V _JAVA_VENDOR_LIST_REGEXP USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(_JAVA_OS_LIST_REGEXP)
+_JAVA_OS_LIST_REGEXP!=		make -V _JAVA_OS_LIST_REGEXP USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(_JAVA_PORTS_INSTALLED)
+_JAVA_PORTS_INSTALLED!=		make -V _JAVA_PORTS_INSTALLED USE_JAVA=1 -f ${PORTSDIR}/Mk/bsd.port.mk
+.endif
+
+.if !defined(UID)
 UID!=	${ID} -u
+.endif
+
 .if exists(${LOCALBASE}/sbin/pkg_info)
 PKG_INFO?=	${LOCALBASE}/sbin/pkg_info
 .else
 PKG_INFO?=	/usr/sbin/pkg_info
 .endif
+.if !defined(PKGINSTALLVER)
 PKGINSTALLVER!=	${PKG_INFO} -P 2>/dev/null | ${SED} -e 's/.*: //'
-
-.if !defined(OPSYS)
-OPSYS!=	${UNAME} -s
 .endif

+.endif
+
+INDEXDIR?=	${PORTSDIR}
+INDEXFILE?=	INDEX-${OSVERSION:C/([0-9]).*/\1/}
+
 # local customization of the ports tree
 .if exists(${.CURDIR}/Makefile.local)
 .include "${.CURDIR}/Makefile.local"
@ -307,6 +362,8 @@ README.html:
 		> $@
 	@${RM} -f $@.tmp $@.tmp2 $@.tmp3 $@.tmp4

+# Pass in the cached invariant variables to child makes.
+# XXX Why are we trying to escape these characters using regexps and not using ':Q'?
 .if !defined(NOPRECIOUSMAKEVARS)
 .MAKEFLAGS: \
 	ARCH="${ARCH:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \
@ -314,7 +371,16 @@ README.html:
 	OSREL="${OSREL:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \
 	OSVERSION="${OSVERSION:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \
 	UID="${UID:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \
-	PKGINSTALLVER="${PKGINSTALLVER:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}"
+	PKGINSTALLVER="${PKGINSTALLVER:S/"/"'"'"/g:S/\$/\$\$/g:S/\\/\\\\/g}" \
+	HAVE_COMPAT_IA32_KERN="${HAVE_COMPAT_IA32_KERN}" \
+	CONFIGURE_MAX_CMD_LEN="${CONFIGURE_MAX_CMD_LEN}" \
+	PYTHON_DEFAULT_VERSION="${PYTHON_DEFAULT_VERSION}" \
+	PYTHON_DEFAULT_PORTVERSION="${PYTHON_DEFAULT_PORTVERSION}" \
+	PYTHONBASE="${PYTHONBASE}" \
+	_JAVA_VERSION_LIST_REGEXP="${_JAVA_VERSION_LIST_REGEXP}" \
+	_JAVA_VENDOR_LIST_REGEXP="${_JAVA_VENDOR_LIST_REGEXP}" \
+	_JAVA_OS_LIST_REGEXP="${_JAVA_OS_LIST_REGEXP}" \
+	_JAVA_PORTS_INSTALLED="${_JAVA_PORTS_INSTALLED}"
 .endif

 PORTSEARCH_DISPLAY_FIELDS?=name,path,info,maint,index,bdeps,rdeps,www
--- a/Mk/bsd.python.mk
+++ b/Mk/bsd.python.mk
@ -352,14 +352,24 @@ _PYTHON_VERSION=	${_PYTHON_PORTBRANCH} # just to avoid version sanity checking.

 PYTHON_VERSION?=	python${_PYTHON_VERSION}
 PYTHON_CMD?=		${_PYTHON_CMD}
+.if !defined(PYTHONBASE)
 PYTHONBASE!=		(${PYTHON_CMD} -c 'import sys; print sys.prefix' \
 						2> /dev/null || ${ECHO_CMD} ${LOCALBASE}) | ${TAIL} -1
+.endif
 DEPENDS_ARGS+=		PYTHON_VERSION=${PYTHON_VERSION}
+
+# We can only use the cached version if we are using the default python version.  Otherwise it
+# should point to some other version we have installed, according to the port USE_PYTHON
+# specification
+.if !defined(PYTHON_DEFAULT_PORTVERSION) || (${PYTHON_VERSION} != ${PYTHON_DEFAULT_VERSION})
 _PYTHON_PORTVERSION!=	(${PYTHON_CMD} -c 'import string, sys; \
 							print string.split(sys.version)[0].replace("b",".b")' 2> /dev/null) | ${TAIL} -1
 .if !defined(PYTHON_NO_DEPENDS) && !empty(_PYTHON_PORTVERSION)
 PYTHON_PORTVERSION=	${_PYTHON_PORTVERSION}
 .endif
+.elif defined(PYTHON_DEFAULT_PORTVERSION)
+PYTHON_PORTVERSION=	${PYTHON_DEFAULT_PORTVERSION}
+.endif

 # Propagate the chosen python version to submakes.
 .MAKEFLAGS:	PYTHON_VERSION=python${_PYTHON_VERSION}
@ -440,7 +450,9 @@ MAKE_ENV+=						PYTHONPATH=${PYEASYINSTALL_SITELIBDIR}
 .endif

 .if defined(PYEASYINSTALL_ARCHDEP)
+.if !defined(_OSRELEASE)
 _OSRELEASE!=					${UNAME} -r
+.endif
 PYEASYINSTALL_OSARCH?=			-${OPSYS:L}-${_OSRELEASE}-${ARCH}
 .endif
 PYEASYINSTALL_EGG?=				${PYDISTUTILS_PKGNAME:C/[^A-Za-z0-9.]+/_/g}-${PYDISTUTILS_PKGVERSION:C/[^A-Za-z0-9.]+/_/g}-${PYTHON_VERSION:S/thon//}${PYEASYINSTALL_OSARCH}.egg
--- a/Tools/make_index
+++ b/Tools/make_index
@ -97,6 +97,8 @@ chomp($pwd = `pwd`);
 # Read each line of output generated by the 'index' target.
 while (<>) {
    chomp;
+    s/\015$//;
+
    my @f = split(/\|/);

  # Force to canonical form.