Require pbench-top|backup-dir in pbench-server.cfg

Remove pbench-top-dir and pbench-backup-dir from the default
configuration file so that they have to be explicitly defined for a
given pbench-server.  We then default pbench-local-dir to be the same as
pbench-top-dir.

Move BDIR definition out of pbench-base.sh because only
pbench-backup-tarballs and pbench-verify-backup-tarballs use it.

Update the pbench-server.cfg-example file to show that pbench-backup-dir
needs to be a directory and not a remote host reference, and add an
explicit definition of pbench-backup-dir to the unit test config files.

Update pbench-backup-tarballs and pbench-verify-backup-tarballs to
refine the error handling around the backup directory configuration.

Update a comment in pbench-server-activate-create-crontab to use an
example hostname, and use the satellite-prefix value in the lock file
name to avoid repeating ourselves.

server/pbench/bin/gold/test-6.3.txt

@@ -55,7 +55,7 @@ drwxrwxr-x          - logs/pbench-audit-server
 -rw-rw-r--          0 logs/pbench-audit-server/pbench-audit-server.error
 -rw-rw-r--          0 logs/pbench-audit-server/pbench-audit-server.log
 drwxrwxr-x          - logs/pbench-backup-tarballs
--rw-rw-r--        140 logs/pbench-backup-tarballs/pbench-backup-tarballs.error
+-rw-rw-r--        199 logs/pbench-backup-tarballs/pbench-backup-tarballs.error
 -rw-rw-r--         30 logs/pbench-backup-tarballs/pbench-backup-tarballs.log
 drwxrwxr-x          - pbench-move-results-receive
 drwxrwxr-x          - pbench-move-results-receive/fs-version-001
@@ -93,7 +93,7 @@ drwxrwxr-x          - quarantine/md5-002
 drwxrwxr-x          - tmp
 --- pbench-satellite-local tree state
 +++ pbench log file contents
-/var/tmp/pbench-test-server/pbench-local/logs/pbench-backup-tarballs/pbench-backup-tarballs.error:pbench-backup-tarballs: Specified backup directory does not resolve to a directory, /var/tmp/pbench-test-server/pbench-local/archive.backup
+/var/tmp/pbench-test-server/pbench-local/logs/pbench-backup-tarballs/pbench-backup-tarballs.error:pbench-backup-tarballs: Specified backup directory, /var/tmp/pbench-test-server/pbench-local/archive.backup, does not resolve (/var/tmp/pbench-test-server/pbench-local/archive.backup) to a directory
 /var/tmp/pbench-test-server/pbench-local/logs/pbench-backup-tarballs/pbench-backup-tarballs.log:start-1900-01-01T00:00:00-UTC
 --- pbench log file contents
 +++ test-execution.log file contents

server/pbench/bin/pbench-backup-tarballs

@@ -16,7 +16,7 @@ case $opts in
         ;;
 esac
 
-# TOP, ARCHIVE, BDIR are defined by the base file
+# ARCHIVE and PBENCH_ENV are defined by the base file
 . $dir/pbench-base.sh
 
 tmp=$TMP/${PROG}.$$
@@ -42,12 +42,14 @@ if [[ ! -d "$rla" ]]; then
     log_exit "The ARCHIVE directory does not resolve to a directory, $ARCHIVE"
 fi
 
-rlb=$(readlink -f $BDIR)
+backup=$(getconf.py pbench-backup-dir pbench-server)
+test ! -z $backup || log_exit "Unspecified backup directory, no pbench-backup-dir config in pbench-server section"
+rlb=$(readlink -f $backup)
 if [[ $? -ne 0 ]]; then
-    log_exit "Specified backup directory does not resolve to a real location, $BDIR"
+    log_exit "Specified backup directory does not resolve to a real location, $backup"
 fi
 if [[ ! -d "$rlb" ]]; then
-    log_exit "Specified backup directory does not resolve to a directory, $BDIR"
+    log_exit "Specified backup directory, $backup, does not resolve ($rlb) to a directory"
 fi
 
 # exclude the subdirs of symlinks
@@ -58,7 +60,7 @@ if [ -z "$(find $rla -maxdepth 0 -type d -empty 2>/dev/null)" ] ;then
     rsync -va --stats $exclude $rla/ $rlb | tee -a $stats
     rsync_sts=$?
     if [[ $rsync_sts -ne 0 ]]; then
-        echo "$PROG: rsync failed with code: $rsync_sts, $BDIR"
+        echo "$PROG: rsync failed with code: $rsync_sts, $rlb"
     fi
 fi
 

server/pbench/bin/pbench-base.sh

@@ -48,7 +48,6 @@ export LOGSDIR=$(getconf.py pbench-logs-dir pbench-server)
 test -d $LOGSDIR || doexit "Bad LOGSDIR=$LOGSDIR"
 
 # Optional
-BDIR=$(getconf.py pbench-backup-dir pbench-server)
 PBENCH_ENV=$(getconf.py environment pbench-server)
 
 if [[ -z "$_PBENCH_SERVER_TEST" ]]; then

server/pbench/bin/pbench-server-activate-create-crontab

@@ -96,10 +96,11 @@ function crontab_normal() {
 # and the [satellite-testing] section looks like this:
 #
 # [satellite-testing]
-# satellite-host = dhcp31-38.perf.lab.eng.bos.redhat.com
+# satellite-host = dhcp31-38.example.com
 # satellite-prefix = TESTING
-# satellite-lock = pbench-sync-satellite-dhcp31-38.lock
+# satellite-lock = pbench-sync-satellite-%(satellite-prefix)s.lock
 # satellite-archive = /pbench/archive/fs-version-001
+# satellite-opt = /opt/pbench-server
 #
 # the (admittedly obscure) last line substitutes for $SATELLITE_HOST the
 # value of satellite-host in the section and similarly for all the other

server/pbench/bin/pbench-verify-backup-tarballs

@@ -36,14 +36,15 @@ case $opts in
         ;;
 esac
 
-# TOP, ARCHIVE, BDIR, TMP, LOGSDIR defined in pbench-base.sh
+# TOP, ARCHIVE, TMP, LOGSDIR, and PBENCH_ENV are defined in pbench-base.sh
 . $dir/pbench-base.sh
 
 test -d $ARCHIVE || doexit "Bad archive: $ARCHIVE"
-test -d $BDIR || doexit "Bad backup directory: $BDIR"
-
 primary=$ARCHIVE
-backup=$BDIR
+
+backup=$(getconf.py pbench-backup-dir pbench-server)
+test ! -z $backup || doexit "Unspecified backup directory, no pbench-backup-dir config in pbench-server section"
+test -d $backup || doexit "Bad backup directory: $backup"
 
 # work files
 controllers=$TMP/$PROG/controllers.$$

server/pbench/bin/state/config/pbench-server.cfg

@@ -11,6 +11,7 @@ environment = unit-test
 admin-email = [email protected]
 pbench-top-dir = %(unittest-dir)s/pbench
 pbench-local-dir = %(unittest-dir)s/pbench-local
+pbench-backup-dir = %(pbench-local-dir)s/archive.backup
 # Add role for sync'ing with satellites
 roles = pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
 # Ensure all dispatch states are used
@@ -35,10 +36,10 @@ tasks = pbench-sync
 [satellite-one]
 # NOTE WELL: this satellite host name, pbench-satellite.example.com, is the
 # host name expected by the mock ssh command which triggers the behavior of
-# running the actual commands instead of just echoing them. 
+# running the actual commands instead of just echoing them.
 satellite-host = pbench-satellite.example.com
 satellite-prefix = ONE
-satellite-lock = pbench-sync-satellite-ONE.lock
+satellite-lock = pbench-sync-satellite-%(satellite-prefix)s.lock
 satellite-archive = %(unittest-dir)s/pbench-satellite/archive/fs-version-001
 satellite-opt = %(unittest-dir)s/opt/pbench-server-satellite
 

server/pbench/bin/state/test-5.1.config/pbench-server.cfg

@@ -20,6 +20,7 @@ admin-email = [email protected]
 pbench-top-dir = %(unittest-dir)s/pbench
 pbench-local-dir = %(unittest-dir)s/pbench-local
 pbench-unpack-dir = %(pbench-local-dir)s/incoming
+pbench-backup-dir = %(pbench-local-dir)s/archive.backup
 # Add role for sync'ing with satellites
 roles = pbench-prep, pbench-results, pbench-backup, pbench-sync-satellites
 # Ensure all dispatch states are used
@@ -44,7 +45,7 @@ tasks = pbench-sync
 [satellite-one]
 satellite-host = pbench-satellite.example.com
 satellite-prefix = ONE
-satellite-lock = pbench-sync-satellite-ONE.lock
+satellite-lock = pbench-sync-satellite-%(satellite-prefix)s.lock
 satellite-archive = %(unittest-dir)s/pbench-satellite/archive/fs-version-001
 satellite-opt = %(unittest-dir)s/opt/pbench-server-satellite
 

server/pbench/lib/config/pbench-server-default.cfg

@@ -3,11 +3,10 @@
 ###########################################################################
 # These should be overridden in the env-specific config file.
 default-host = pbench.example.com
-#
-###########################################################################
-
 default-user = pbench
 default-group = pbench
+#
+###########################################################################
 
 # WARNING - the pbench-server.cfg file should provide a definition of
 # install-dir, e.g.:
@@ -30,7 +29,11 @@ mailto=%(admin-email)s
 mailfrom=%(user)s@%(host)s
 
 # See pbench-server-setup documentation for filesystem setup.
-pbench-top-dir = /srv/pbench
+
+# WARNING - the pbench-server.cfg file should provide a definition of
+# pbench-top-dir, e.g.:
+#     pbench-top-dir = /srv/pbench
+# We won't define it here by default to avoid unintended behaviors.
 
 pbench-archive-version = 001
 pbench-archive-dir = %(pbench-top-dir)s/archive/fs-version-%(pbench-archive-version)s
@@ -43,16 +46,26 @@ pbench-results-dir = %(pbench-html-dir)s/results
 pbench-users-dir = %(pbench-html-dir)s/users
 pbench-static-dir = %(pbench-html-dir)s/static
 
-pbench-local-dir = /srv/pbench-local
+# By default the local directory is the same as the top directory. You
+# might want to consider placing the local directory on a separate FS
+# to avoid the temporary files, log files, quarantine, and reception files
+# from competing with disk bandwidth and space of the archive, incoming and
+# results hierarchies.
+pbench-local-dir = %(pbench-top-dir)s
+
 pbench-logs-dir = %(pbench-local-dir)s/logs
 pbench-tmp-dir = %(pbench-local-dir)s/tmp
-pbench-backup-dir = %(pbench-local-dir)s/archive.backup
 # Which client move-results versions we support
 pbench-move-results-receive-versions = 001 002
 pbench-receive-dir-001 = %(pbench-local-dir)s/pbench-move-results-receive/fs-version-001
 pbench-receive-dir-002 = %(pbench-local-dir)s/pbench-move-results-receive/fs-version-002
 pbench-quarantine-dir = %(pbench-local-dir)s/quarantine
 
+# WARNING - the pbench-server.cfg file should provide a definition of
+# pbench-backup-dir, e.g.:
+#     pbench-backup-dir = %(pbench-local-dir)s/archive.backup
+# We won't define it here by default to avoid unintended behaviors.
+
 # Default roles this pbench server takes on, see crontab roles below.
 roles = pbench-prep, pbench-results, pbench-backup
 

server/pbench/lib/config/pbench-server-satellite.cfg.example

@@ -9,6 +9,7 @@ install-dir = /opt/pbench-server
 ## Deployment section
 ###########################################################################
 [pbench-server]
+pbench-top-dir = /srv/pbench
 # For a satellite we override the default roles to only "prep" and "results"
 roles = pbench-prep, pbench-results
 

server/pbench/lib/config/pbench-server.cfg.example

@@ -6,9 +6,10 @@ default-host = pbench.example.com
 install-dir = /opt/pbench-server
 
 [pbench-server]
-# Backup dir may be remote
-backup-host = pbench-backup.example.com
-pbench-backup-dir = %(user)s@%(backup-host)s:/srv/pbench.archive.backup
+pbench-top-dir = /srv/pbench
+pbench-local-dir = /srv/pbench-local
+# Backup dir may be remote NFS mounted
+pbench-backup-dir = /mnt/pbench.archive.backup
 
 ###########################################################################
 # crontab roles
@@ -31,14 +32,14 @@ tasks = pbench-sync
 [satellite-one]
 satellite-host = satellite-one.example.com
 satellite-prefix = ONE
-satellite-lock = pbench-sync-satellite-ONE.lock
+satellite-lock = pbench-sync-satellite-%(satellite-prefix)s.lock
 satellite-archive = /pbench/archive/fs-version-001
 satellite-opt = /opt/pbench-server
 
 [satellite-two]
 satellite-host = satellite-two.example.com
 satellite-prefix = TWO
-satellite-lock = pbench-sync-satellite-TWO.lock
+satellite-lock = pbench-sync-satellite-%(satellite-prefix)s.lock
 satellite-archive = /pbench/archive/fs-version-001
 satellite-opt = /opt/pbench-server