From 86571b9c01dee210fcc6a6b0a74b8cb6ee45ba90 Mon Sep 17 00:00:00 2001
From: Alan Somers <asomers@FreeBSD.org>
Date: Sat, 1 Apr 2017 04:42:35 +0000
Subject: [PATCH] Consolidate random sleeps in periodic scripts

Multiple periodic scripts sleep for a random amount of time in order to
mitigate the thundering herd problem. This is bad, because the sum of
multiple uniformly distributed random variables approaches a normal
distribution, so the problem isn't mitigated as effectively as it would be
with a single sleep.

This change creates a single configurable anticongestion sleep. periodic
will only sleep if at least one script requires it, and it will never sleep
more than once per invocation. It also won't sleep if periodic was run
interactively, fixing an unrelated longstanding bug.

PR:		217055
PR:		210188
Reviewed by:	cy
MFC after:	3 weeks
Differential Revision:	https://reviews.freebsd.org/D10211
---
 etc/defaults/periodic.conf           | 16 ++++++++++++++--
 etc/periodic/daily/480.leapfile-ntpd | 13 +++----------
 share/man/man5/periodic.conf.5       |  6 +++++-
 usr.sbin/periodic/periodic.sh        |  7 +++++++
 4 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/etc/defaults/periodic.conf b/etc/defaults/periodic.conf
index 16de38edcd88..b3be114fd98a 100644
--- a/etc/defaults/periodic.conf
+++ b/etc/defaults/periodic.conf
@@ -22,6 +22,8 @@ periodic_conf_files="/etc/periodic.conf /etc/periodic.conf.local"
 # periodic script dirs
 local_periodic="/usr/local/etc/periodic"
 
+# Max time to sleep to avoid causing congestion on download servers
+anticongestion_sleeptime=3600
 
 # Daily options
 
@@ -136,8 +138,6 @@ daily_status_mail_rejects_shorten="NO"			# Shorten output
 
 # 480.leapfile-ntpd
 daily_ntpd_leapfile_enable="YES"			# Fetch NTP leapfile
-daily_ntpd_avoid_congestion="YES"			# Avoid congesting
-							# leapfile sources
 
 # 480.status-ntpd
 daily_status_ntpd_enable="NO"				# Check NTP status
@@ -307,6 +307,18 @@ security_status_tcpwrap_period="daily"
 if [ -z "${source_periodic_confs_defined}" ]; then
         source_periodic_confs_defined=yes
 
+	# Sleep for a random amount of time in order to mitigate the thundering
+	# herd problem of multiple hosts running periodic simultaneously.
+	# Will not sleep when used interactively.
+	# Will sleep at most once per invocation of periodic
+	anticongestion() {
+		[ -n "$PERIODIC_IS_INTERACTIVE" ] && return
+		if [ -f "$PERIODIC_ANTICONGESTION_FILE" ]; then
+			rm -f $PERIODIC_ANTICONGESTION_FILE
+			sleep `jot -r 1 0 ${anticongestion_sleeptime}`
+		fi
+	}
+
 	# Compatibility with old daily variable names.
 	# They can be removed in stable/11.
 	security_daily_compat_var() {
diff --git a/etc/periodic/daily/480.leapfile-ntpd b/etc/periodic/daily/480.leapfile-ntpd
index 8a3836427b4e..6b2a1f33f1b7 100755
--- a/etc/periodic/daily/480.leapfile-ntpd
+++ b/etc/periodic/daily/480.leapfile-ntpd
@@ -13,16 +13,9 @@ fi
 
 case "$daily_ntpd_leapfile_enable" in
     [Yy][Ee][Ss])
-	case "$daily_ntpd_avoid_congestion" in
-    	[Yy][Ee][Ss])
-	    # Avoid dogpiling
-	    (sleep $(jot -r 1 0 3600); service ntpd onefetch) &
-	    ;;
-	*)
-	    service ntpd onefetch
-	    ;;
-	esac
-	;;
+        anticongestion
+        service ntpd onefetch
+        ;;
 esac
 
 exit $rc
diff --git a/share/man/man5/periodic.conf.5 b/share/man/man5/periodic.conf.5
index cb9a8b06b39c..d0beb1d1bd8e 100644
--- a/share/man/man5/periodic.conf.5
+++ b/share/man/man5/periodic.conf.5
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 26, 2015
+.Dd March 31, 2015
 .Dt PERIODIC.CONF 5
 .Os
 .Sh NAME
@@ -133,6 +133,10 @@ respectively.
 Refer to the
 .Xr periodic 8
 manual page for how script return codes are interpreted.
+.It Va anticongestion_sleeptime
+.Pq Vt int
+The maximum number of seconds to randomly sleep in order to smooth bursty loads
+on a shared resource, such as a download mirror.
 .El
 .Pp
 The following variables are used by the standard scripts that reside in
diff --git a/usr.sbin/periodic/periodic.sh b/usr.sbin/periodic/periodic.sh
index b2605200f79e..bef87f89dde6 100644
--- a/usr.sbin/periodic/periodic.sh
+++ b/usr.sbin/periodic/periodic.sh
@@ -76,6 +76,12 @@ fi
 shift
 arg=$1
 
+if [ -z "$PERIODIC_ANTICONGESTION_FILE" ] ; then
+	export PERIODIC_ANTICONGESTION_FILE=`mktemp ${TMPDIR:-/tmp}/periodic.anticongestion.XXXXXXXXXX`
+fi
+if tty > /dev/null 2>&1; then
+	export PERIODIC_IS_INTERACTIVE=1
+fi
 tmp_output=`mktemp ${TMPDIR:-/tmp}/periodic.XXXXXXXXXX`
 context="$PERIODIC"
 export PERIODIC="$arg${PERIODIC:+ }${PERIODIC}"
@@ -141,3 +147,4 @@ esac
 } | output_pipe $arg "$context"
 
 rm -f $tmp_output
+rm -f $PERIODIC_ANTICONGESTION_FILE