#!/usr/bin/perl
#
# $Revision: 1.156.2.6 $
# Copyright 2012-2015 Teleflora
#
# Script to perform server to server backups of an RTI and/or Daisy system.
#

use strict;
use warnings;
use POSIX;
use Getopt::Long;
use English;
use Net::SMTP;
use File::Spec;
use File::Basename;
use File::Temp qw(tempfile);
use File::stat;
use Sys::Hostname;
use Fcntl qw(:flock SEEK_END);

use lib '/usr/local/ostools/modules';
use OSTools::Platform;
use OSTools::Hardware;
use OSTools::AppEnv;
use OSTools::Filesys;


my $CVS_REVISION = '$Revision: 1.156.2.6 $';
my $TIMESTAMP = POSIX::strftime("%Y%m%d%H%M%S", localtime());
my $PROGNAME = basename($0);


############################
###                      ###
###      DEFINITIONS     ###
###                      ###
############################

# exit status values
my $EXIT_OK = 0;
my $EXIT_COMMAND_LINE = 1;
my $EXIT_MUST_BE_ROOT = 2;
my $EXIT_ROOTDIR = 3;
my $EXIT_TOOLSDIR = 4;
my $EXIT_BACKUP_DEVICE_NOT_FOUND = 5;
my $EXIT_USB_DEVICE_NOT_FOUND = 6;
my $EXIT_BACKUP_TYPE = 7;
my $EXIT_MOUNT_ERROR = 8;
my $EXIT_RESTORE = 9;
my $EXIT_DEVICE_NOT_SPECIFIED = 10;
my $EXIT_DEVICE_VERIFY = 11;
my $EXIT_USB_DEVICE_UNSUPPORTED = 12;
my $EXIT_CRON_JOB_FILE = 13;
my $EXIT_DEF_CONFIG_FILE = 14;
my $EXIT_FORMAT = 15;
my $EXIT_LIST_UNSUP = 16;
my $EXIT_LIST = 17;
my $EXIT_SIGINT = 18;
my $EXIT_RSYNC_ACCOUNT = 19;
my $EXIT_SSH_GENERATE_KEYS = 20;
my $EXIT_SSH_GET_PUBLIC_KEY = 21;
my $EXIT_SSH_GET_PRIVATE_KEY = 22;
my $EXIT_SSH_SUDO_CONF = 23;
my $EXIT_SSH_TUNNEL_OPEN = 24;
my $EXIT_SSH_TUNNEL_CLOSE = 25;
my $EXIT_SSH_COPY_PUBLIC_KEY = 26;
my $EXIT_SSH_ID_FILE = 27;
my $EXIT_INSTALL_SECONDARY = 28;
my $EXIT_CLOUD_ACCOUNT_NAME = 29;
my $EXIT_GENERATE_PERMS = 30;
my $EXIT_UPLOAD_PERMS = 31;
my $EXIT_RESTORE_PERMS = 32;
my $EXIT_DOWNLOAD_PERMS = 33;
my $EXIT_PERM_FILE_MISSING = 34;
my $EXIT_RESTORE_DEVICE = 35;
my $EXIT_LOCK_SETUP = 39;
my $EXIT_LOCK_ACQUISITION = 40;
my $EXIT_BACKUP_DEVICE_CONFLICT = 41;
my $EXIT_PLATFORM = 42;
my $EXIT_RSYNC_ERROR = 43;
my $EXIT_USERS_INFO_SAVE = 44;
my $EXIT_PSERVER_INFO_SAVE = 45;
my $EXIT_PSERVER_CLOISTER_FILES_SAVE = 46;
my $EXIT_XFERDIR_WRITE_ERROR = 50;
my $EXIT_XFERDIR_MKDIR = 51;
my $EXIT_XFERDIR_RMDIR = 52;
my $EXIT_INFODIR_MKDIR = 53;
my $EXIT_INFODIR_RMDIR = 54;
my $EXIT_USERSDIR_MKDIR = 55;
my $EXIT_USERSDIR_RMDIR = 56;
my $EXIT_TOP_LEVEL_MKDIR = 57;
my $EXIT_INSTALL_PSERVER_INFO_FILE = 58;
my $EXIT_DOVE_SERVER_MISSING = 59;
my $EXIT_DOVE_SERVER_SAVE_EXISTS = 60;
my $EXIT_DOVE_SERVER_RM = 61;
my $EXIT_CLOISTERDIR_MKDIR = 62;
my $EXIT_CLOISTERDIR_RMDIR = 63;
my $EXIT_RUNTIME_CLEANUP = 64;
my $EXIT_COULD_NOT_EXECUTE = 70;
my $EXIT_FROM_SIGNAL = 71;
my $EXIT_LOGFILE_SETUP = 72;
my $EXIT_NET_IPADDR = 73;
my $EXIT_NET_NETMASK = 74;
my $EXIT_NET_GATEWAY = 75;
my $EXIT_UNKNOWN = 99;

my %ExitTable = (
    $EXIT_OK => "Exit OK",
    $EXIT_COMMAND_LINE => "Command line error",
    $EXIT_MUST_BE_ROOT => "Must be root to run script",
    $EXIT_ROOTDIR => "Directory specified with --rootdir does not exist",
    $EXIT_TOOLSDIR => "OSTools directory does not exist",
    $EXIT_BACKUP_DEVICE_NOT_FOUND => "Backup device not found",
    $EXIT_USB_DEVICE_NOT_FOUND => "USB backup device not found",
    $EXIT_BACKUP_TYPE => "Unknown backup type",
    $EXIT_MOUNT_ERROR => "Error mounting backup device",
    $EXIT_RESTORE => "Restore error",
    $EXIT_DEVICE_NOT_SPECIFIED => "Backup device not specified",
    $EXIT_DEVICE_VERIFY => "Backup device can not be verified",
    $EXIT_USB_DEVICE_UNSUPPORTED => "Unsupported USB backup device",
    $EXIT_CRON_JOB_FILE => "Error installing cron job",
    $EXIT_DEF_CONFIG_FILE => "Error installing default config file",
    $EXIT_FORMAT => "Error formatting the backup device",
    $EXIT_LIST_UNSUP => "Unknown list type",
    $EXIT_LIST => "Error with list",
    $EXIT_SIGINT => "Process received signal SIGINT",
    $EXIT_RSYNC_ACCOUNT => "Error installing or removing rsync account",
    $EXIT_SSH_GENERATE_KEYS => "Error generating ssh key pair",
    $EXIT_SSH_GET_PUBLIC_KEY => "Error getting the ssh public key",
    $EXIT_SSH_GET_PRIVATE_KEY => "Error getting the private key",
    $EXIT_SSH_SUDO_CONF => "Error sudoers config file",
    $EXIT_SSH_TUNNEL_OPEN => "Error opening the ssh tunnel socket",
    $EXIT_SSH_TUNNEL_CLOSE => "Error closing the ssh tunnel socket",
    $EXIT_SSH_COPY_PUBLIC_KEY => "Error copying the public key",
    $EXIT_SSH_ID_FILE => "Error with ssh id file",
    $EXIT_CLOUD_ACCOUNT_NAME => "Error with cloud account name",
    $EXIT_GENERATE_PERMS => "Error generating perm files",
    $EXIT_UPLOAD_PERMS => "Error uploading perm files",
    $EXIT_RESTORE_PERMS => "Error restoring perm files",
    $EXIT_DOWNLOAD_PERMS => "Error downloading perm files",
    $EXIT_PERM_FILE_MISSING => "Perm file missing",
    $EXIT_LOCK_SETUP => "Error setting up process lock",
    $EXIT_LOCK_ACQUISITION => "Could not acquire process lock",
    $EXIT_BACKUP_DEVICE_CONFLICT => "Backup device conflict",
    $EXIT_PLATFORM => "Unsupported platform",
    $EXIT_RSYNC_ERROR => "Non-zero exit status from rsync command",
    $EXIT_USERS_INFO_SAVE => "Could not save users info",
    $EXIT_PSERVER_INFO_SAVE => "Could not save pserver info",
    $EXIT_PSERVER_CLOISTER_FILES_SAVE => "Cound not save pserver cloister files",
    $EXIT_XFERDIR_WRITE_ERROR => "Error writing to transfer dir",
    $EXIT_XFERDIR_MKDIR => "Error from mkdir of transfer dir",
    $EXIT_XFERDIR_RMDIR => "Error removing transfer dir",
    $EXIT_INFODIR_MKDIR => "Error from mkdir of info dir",
    $EXIT_INFODIR_RMDIR => "Error removing info dir",
    $EXIT_USERSDIR_MKDIR => "Error making users info dir",
    $EXIT_USERSDIR_RMDIR => "Error removing users info dir",
    $EXIT_TOP_LEVEL_MKDIR => "Error making top level project dir",
    $EXIT_INSTALL_PSERVER_INFO_FILE => "Could not get production server info file",
    $EXIT_DOVE_SERVER_MISSING => "Dover server script missing",
    $EXIT_DOVE_SERVER_SAVE_EXISTS => "Saved Dove server script exists",
    $EXIT_CLOISTERDIR_MKDIR => "Error making pserver cloister dir",
    $EXIT_CLOISTERDIR_RMDIR => "Error removing pserver cloister dir",
    $EXIT_COULD_NOT_EXECUTE => "Could not execute sub program",
    $EXIT_FROM_SIGNAL => "From signal",
    $EXIT_LOGFILE_SETUP => "Log file setup error",
    $EXIT_NET_IPADDR => "Network IP Addr error",
    $EXIT_NET_NETMASK => "Network netmask error",
    $EXIT_NET_GATEWAY => "Network gateway error",
    $EXIT_UNKNOWN => "Unknown reason for exit",
);


my $EMPTY_STR = q{};
my @EMPTY_LIST = ();

#
# Device types are:
# 1) "passport"    - a Western Digital Passport USB disk
# 2) "rev"         - an IOmega Rev drive
# 3) "usb"         - a USB disk with Teleflora label
# 4) "image"	   - an image file
# 5) "server"      - ip address of backup server
# 6) "file system" - locally mounted file system
# 7) "block"       - either "passport", "rev", or "usb"
# 8) "show-only"   - special device, same thing as "--show-only"
# 9) "cloud"       - special device, same thing as "--cloud"
#
my $DEVTYPE_PASSPORT    = "passport";
my $DEVTYPE_REV         = "rev";
my $DEVTYPE_USB         = "usb";
my $DEVTYPE_IMAGE       = "image";
my $DEVTYPE_SERVER      = "server";
my $DEVTYPE_FILE_SYSTEM = "file system";
my $DEVTYPE_BLOCK       = "block";
my $DEVTYPE_SHOW_ONLY   = "show-only";
my $DEVTYPE_CLOUD       = "cloud";
my $DEVTYPE_DEVICE      = "device";
my $DEVTYPE_ANY         = "any";
my $DEVTYPE_UNK         = "unknown";

# defaults
my $DEF_PROJ_NAME = "tfrsync";

# RTI locations
my $RTI_TOPDIR            = "/usr2";
my $RTIDIR                = $RTI_TOPDIR . '/' . "bbx";
my $RTI_BINDIR            = $RTIDIR . '/' . "bin";
my $RTI_CONFIGDIR         = $RTIDIR . '/' . "config";
my $RTI_LOGDIR            = $RTIDIR . '/' . "log";
my $RTI_TOOLSDIR          = $RTI_TOPDIR . '/' . 'ostools';
my $RTI_TOOLS_BINDIR      = $RTI_TOOLSDIR . '/' . 'bin';
my $RTI_SHOPCODE_FILE     = $RTI_CONFIGDIR . '/'. "dove.ini";

# Daisy locations
my $DAISY_TOPDIR          = "/d";
my $DAISYDIR              = $DAISY_TOPDIR . '/' . "daisy";
my $DAISY_BINDIR          = $DAISYDIR . '/' . "bin";
my $DAISY_CONFIGDIR       = $DAISYDIR . '/' . "config";
my $DAISY_LOGDIR          = $DAISYDIR . '/' . "log";
my $DAISY_TOOLSDIR        = $DAISY_TOPDIR . '/' . 'ostools';
my $DAISY_TOOLS_BINDIR    = $DAISY_TOOLSDIR . '/' . 'bin';
my $DAISY_SHOPCODE_FILE   = $DAISYDIR . '/' . "dovectrl.pos";

my $DEF_TFSUPPORT_ACCOUNT_NAME = 'tfsupport';
my $DEF_RTI_GROUP_NAME         = 'rti';
my $DEF_RTI_ADMIN_GROUP_NAME   = 'rtiadmins';
my $DEF_DAISY_GROUP_NAME       = 'daisy';
my $DEF_DAISY_ADMIN_GROUP_NAME = 'dsyadmins';

my $MOUNTPOINT   = "/mnt/backups";

my $DEVICE_IMAGE_FILE_MIN = 1048576; # 1 MB in bytes

#
# there is a log file for each of the backup classes and
# the default value contains variables used with "strftime()"
# to put the day of the month and the time in the file name
# during log file setup.
#
my $DEF_RTI_LOG_DIR      = $RTI_LOGDIR;
my $DEF_DAISY_LOG_DIR    = $DAISY_LOGDIR;
# in case POS log directory does not exist
my $DEF_ALT_LOGFILE_DIR  = '/tmp';
# cloud log file name
my $DEF_LOGFILE_CLOUD    = $DEF_PROJ_NAME . '-cloud-Day_%d.log';
# server log file name
my $DEF_LOGFILE_SERVER   = $DEF_PROJ_NAME . '-server-Day_%d.log';
# device log file name
my $DEF_LOGFILE_DEVICE   = $DEF_PROJ_NAME . '-device-Day_%d.log';
# default log file name
my $DEF_LOGFILE          = $DEF_PROJ_NAME . '-Day_%d.log';
# summary log file name
my $DEF_SUMMARY_LOGFILE  = $DEF_PROJ_NAME . '-summary.log';
# debug log file name
my $DEF_DEBUG_LOGFILE    = $DEF_PROJ_NAME . '-debug.log';
# rsync stats log file name
my $DEF_RSYNC_STATS_LOG  = $DEF_PROJ_NAME . '-rsync-stats.log';

# wait this number of seconds to lock log file
my $WAIT_FOR_LOG_FILE_LOCK = 30;

my $DEF_ROOTDIR = '/';

my $DEF_TOOLS_CONFIG_DIR_NAME  = "config";
my $DEF_CONFIG_FILE_EXT        = ".conf";
my $DEF_CONFIG_FILENAME        = $DEF_PROJ_NAME . $DEF_CONFIG_FILE_EXT;
my $DEF_CONFIG_DIR_NAME        = $DEF_PROJ_NAME . ".d";
my $DEF_RTI_CONFIG_FILE_PATH   = $RTI_CONFIGDIR . '/' . $DEF_CONFIG_FILENAME;
my $DEF_DAISY_CONFIG_FILE_PATH = $DAISY_CONFIGDIR . '/' . $DEF_CONFIG_FILENAME;

my $DEF_SUDOERS_CONTENT_FILE   = $DEF_PROJ_NAME . $DEF_CONFIG_FILE_EXT;

my $DEF_RTI_RSYNC_BU_DIR   = $RTI_TOPDIR . '/' . $DEF_PROJ_NAME;
my $DEF_DAISY_RSYNC_BU_DIR = $DAISY_TOPDIR . '/' . $DEF_PROJ_NAME;

my $CRON_JOB_SERVER_PATH   = "/etc/cron.d" . '/' . $DEF_PROJ_NAME . "-server";
my $CRON_JOB_CLOUD_PATH    = "/etc/cron.d" . '/' . $DEF_PROJ_NAME . "-cloud";
my $CRON_JOB_DEVICE_PATH   = "/etc/cron.d" . '/' . $DEF_PROJ_NAME . "-device";
my $CRON_JOB_TYPE_SERVER   = $DEVTYPE_SERVER;
my $CRON_JOB_TYPE_CLOUD    = $DEVTYPE_CLOUD; 
my $CRON_JOB_TYPE_DEVICE   = $DEVTYPE_DEVICE;

my @CRON_JOB_TYPES = (
    $CRON_JOB_TYPE_SERVER,
    $CRON_JOB_TYPE_CLOUD,
    $CRON_JOB_TYPE_DEVICE,
);

my $DEF_PRIMARY_SERVER = '192.168.1.21';
my $DEF_RSYNC_SERVER = '192.168.1.22';

my $DEF_RSYNC_ACCOUNT = $DEF_PROJ_NAME;
my $DEF_RSYNC_ACCOUNT_FULL_NAME = $DEF_PROJ_NAME . ' user';

# default rsync command timeout, value is in seconds.
# this value added to backups and restores via the
# rsync "--timeout" command line option.
# a value of 0 means no timeout
my $DEF_RSYNC_TIMEOUT = 600;

my $DEF_RSYNC_COMPRESSION = 0;

my $DEF_CLOUD_SERVER = 'rticloud.homelinux.com';

my $SSH_KEY_FILENAME = 'id_rsa';
my $SSH_KEY_FILENAME_PUBLIC = 'id_rsa.pub';
my $SSH_AUTH_KEYS_FILENAME = 'authorized_keys';
my $SSH_KEY_LEN = 2048;
my $SSH_KEY_TYPE = 'rsa';
my $SSH_PUBLIC_KEY_CACHEDIR = $DEF_PROJ_NAME . '_public_key';
my $DEF_SSH_SOCKET_DIR_PATH = "/var/run";
my $DEF_SSH_SOCKET_PREFIX = $DEF_PROJ_NAME . '-';
my $DEF_SSH_SOCKET_EXT = ".socket";

# number of times to retry opening ssh tunnel
my $DEF_SSH_TUNNEL_RETRIES = 3;

my $KEY_TYPE_PRIVATE = 1;
my $KEY_TYPE_PUBLIC = $KEY_TYPE_PRIVATE + 1;

my $TELEFLORA_FS_LABEL_PATTERN = 'TFBUDSK-\d{8}';
#
# The name of the rtibackup.pl setup file - it is written to the
# backup device any time it is formatted via "--format".
#
my $FORMAT_FILE = "teleflora-formatted.txt";

my $LOCKFILE_DIR         = "/var/lock";
my $LOCKFILE_SUFFIX      = ".lock";
my $LOCKFILE_TYPE_SERVER = $DEVTYPE_SERVER;
my $LOCKFILE_TYPE_CLOUD  = $DEVTYPE_CLOUD;
my $LOCKFILE_TYPE_DEVICE = $DEVTYPE_DEVICE;
my $LOCKFILE_TYPE_UNK    = $DEVTYPE_UNK;

my $PERM_FILE_SUFFIX = "-perms.txt";

my $RSYNCSTATS_TEMPLATE = $DEF_PROJ_NAME . "-rsyncstats-XXXXXXX";

# name of directory used to hold info to be transferred
# between production server and backup server...
my $PSERVER_XFER_DIR         = "pserver_xfer.d";

# name of dir to hold production server info file
my $PSERVER_INFO_DIR         = "pserver_info.d";

# name of the file to hold production server info
my $PSERVER_INFO_FILE        = "pserver_info.txt";

# name of dir to hold the restored production server info file
my $RESTORED_PSERVER_INFO_DIR = "restored_pserver_info.d";

# name of dir to hold backup server info files
my $BUSERVER_INFO_DIR        = "buserver_info.d";

# name of the file to hold the backup server info
my $BUSERVER_INFO_FILE       = "buserver_info.txt";

# name of dir to hold cloistered files
my $PSERVER_CLOISTER_DIR     = "pserver_cloister.d";

# name of dir to hold POS users info files
my $USERS_INFO_DIR           = "users_info.d";

my $RTI_USERS_LISTING_FILE   = "rti_users_listing.txt";
my $RTI_USERS_SHADOW_FILE    = "rti_users_shadow.txt";
my $DAISY_USERS_LISTING_FILE = "daisy_users_listing.txt";
my $DAISY_USERS_SHADOW_FILE  = "daisy_users_shadow.txt";

#
# keys for the pserver info file
#
my $SERVER_INFO_PLATFORM  = "platform";
my $SERVER_INFO_HOSTNAME  = "hostname";
my $SERVER_INFO_IPADDR    = "ipaddr";
my $SERVER_INFO_NETMASK   = "netmask";
my $SERVER_INFO_GATEWAY   = "gateway";

my $BU_TYPE_ALL              = "all";
my $BU_TYPE_USR2             = "usr2";
my $BU_TYPE_DAISY            = "daisy";
my $BU_TYPE_PRINT_CONFIGS    = "printconfigs";
my $BU_TYPE_RTI_CONFIGS      = "rticonfigs";
my $BU_TYPE_DAISY_CONFIGS    = "daisyconfigs";
my $BU_TYPE_OS_CONFIGS       = "osconfigs";
my $BU_TYPE_NET_CONFIGS      = "netconfigs";
my $BU_TYPE_USER_CONFIGS     = "userconfigs";
my $BU_TYPE_USER_FILES       = "userfiles";
my $BU_TYPE_LOG_FILES        = "logfiles";

my $BU_TYPE_POS_USERS_INFO   = "posusersinfo";
my $BU_TYPE_PSERVER_INFO     = "pserverinfo";
my $BU_TYPE_PSERVER_CLOISTER = "pservercloister";

my $BU_TYPE_BBXD             = "bbxd";
my $BU_TYPE_BBXPS            = "bbxps";
my $BU_TYPE_SINGLEFILE       = "singlefile";

my $BU_TYPE_POS_LOG_FILES    = "poslogfiles";
my $BU_TYPE_POS_SUMMARY_LOG  = "possummarylog";

my @BACKUP_TYPES = (
    $BU_TYPE_ALL,
    $BU_TYPE_USR2,
    $BU_TYPE_DAISY,
    $BU_TYPE_PRINT_CONFIGS,
    $BU_TYPE_RTI_CONFIGS,
    $BU_TYPE_DAISY_CONFIGS,
    $BU_TYPE_OS_CONFIGS,
    $BU_TYPE_NET_CONFIGS,
    $BU_TYPE_USER_CONFIGS,
    $BU_TYPE_USER_FILES,
    $BU_TYPE_LOG_FILES,
);

my @BACKUP_EQUALS_ALL_BACKUP_TYPES = (
    $BU_TYPE_USR2,
    $BU_TYPE_DAISY,
    $BU_TYPE_PRINT_CONFIGS,
    $BU_TYPE_RTI_CONFIGS,
    $BU_TYPE_DAISY_CONFIGS,
    $BU_TYPE_OS_CONFIGS,
    $BU_TYPE_NET_CONFIGS,
    $BU_TYPE_USER_CONFIGS,
    $BU_TYPE_LOG_FILES,
    $BU_TYPE_USER_FILES,
    $BU_TYPE_PSERVER_CLOISTER,
    $BU_TYPE_POS_USERS_INFO,
    $BU_TYPE_PSERVER_INFO,
);

my @RESTORE_EQUALS_ALL_BACKUP_TYPES = (
    $BU_TYPE_USR2,
    $BU_TYPE_DAISY,
    $BU_TYPE_PRINT_CONFIGS,
    $BU_TYPE_RTI_CONFIGS,
    $BU_TYPE_DAISY_CONFIGS,
    $BU_TYPE_OS_CONFIGS,
    $BU_TYPE_NET_CONFIGS,
    $BU_TYPE_USER_CONFIGS,
    $BU_TYPE_USER_FILES,
    $BU_TYPE_SINGLEFILE,
    $BU_TYPE_PSERVER_CLOISTER,
);

my @UNIQUE_BACKUP_TYPES = (
    $BU_TYPE_POS_USERS_INFO,
    $BU_TYPE_PSERVER_INFO,
    $BU_TYPE_PSERVER_CLOISTER,
    $BU_TYPE_BBXD,
    $BU_TYPE_BBXPS,
    $BU_TYPE_SINGLEFILE,
);

#
# Special backup types are those that DO NOT get copied
# in place.
#
my @SPECIAL_BACKUP_TYPES = (
    $BU_TYPE_NET_CONFIGS,
    $BU_TYPE_OS_CONFIGS,
    $BU_TYPE_USER_CONFIGS,
    $BU_TYPE_USER_FILES,
);

my @RESTORE_TYPES = (
    @BACKUP_TYPES,
    $BU_TYPE_BBXD,
    $BU_TYPE_BBXPS,
    $BU_TYPE_SINGLEFILE,
    $BU_TYPE_PSERVER_CLOISTER,
);

my @DEF_BACKUP_EXCLUDES = qw(
    *.iso
    *.tar
    *.tar.gz
    *.tar.asc
    *.revision
    *.patch
    lost+found
);
my @DEF_RTI_BACKUP_EXCLUDES = qw(
    /usr2/*.iso
    /usr2/*.tar.asc
    /usr2/bbx-*
    /usr2/bbx/*.iso
    /usr2/bbx/bbxt/*
    /usr2/bbx/backups/*
    /usr2/bbx/*.tar.asc
    /usr2/bbx/bbxtmp/*
    /usr2/basis-old
    /usr2/basis/cache/*
);
my @DEF_RTI_RESTORE_EXCLUDES = qw(
    /usr2/bbx/backups/*
    /usr2/bbx/bbxtmp/*
    /usr2/bbx/log/*
    /usr2/basis/log/*
    /usr2/basis/cache/*
);

my @DEF_DAISY_BACKUP_EXCLUDES = qw(
    /d/*.iso
    /d/*.tar.asc
    /d/backup
);

#
# RTI files and directories excluded from from USR2 backup
#
my $RTI_DOVE_CMD         = $RTI_BINDIR . '/' . "doveserver.pl";
my $RTI_BASIS_CACHE_DIR  = $RTI_TOPDIR . '/basis/cache';
my $RTI_DELVCONF         = $RTIDIR . '/delvconf';

# "--retry-backup" cmd line options defaults
my $DEF_RETRY_BACKUP = 0;           # default policy is to not do retries
my $DEF_RETRY_BACKUP_REPS = 3;      # default number of backup retries
my $MAX_RETRY_BACKUP_REPS = 10;     # max value
my $DEF_RETRY_BACKUP_WAIT = 120;    # default seconds to wait between retries
my $MAX_RETRY_BACKUP_WAIT = 3600;   # max value = 1 hour

# network device
my $DEF_NETWORK_DEVICE = "eth0";

# error exit status values of rsync(1)
my $RSYNC_EXIT_STATUS_PARTIAL = 23;
my $RSYNC_EXIT_STATUS_VANISHED = 24;
my $RSYNC_EXIT_STATUS_IO_ERROR = 30;
my $RSYNC_EXIT_STATUS_SSH_ERROR = 255;


############################
###                      ###
###   GLOBAL VARIABLES   ###
###                      ###
############################

my @ARGV_ORIG = @ARGV;

# logfile vars
my $LogfileDir     = $EMPTY_STR;
my $LogfileName    = $EMPTY_STR;
my $LOGFILE        = $EMPTY_STR;
my $SummaryLogfile = $EMPTY_STR;
my $DebugLogfile   = $EMPTY_STR;

my @USERFILES = ();

my @EXCLUDES = ();

my $EMAIL_SERVER = "";
my $EMAIL_USER = "";
my $EMAIL_PASS = "";

my $DeviceType = "";

# path to the existing ostools directory
my $ToolsDir = "";

my $ExitStatus = $EXIT_OK;

# the last exit status of the rsync command for backup operations.
my $RsyncExitStatus = 0;
# true if any rsync timeout error occurred
my $RsyncTimeoutSeen = 0;

my $RsyncStatsSent = 0;
my $RsyncStatsReceived = 0;
my $RsyncStatsRate = 0;

# list of rsync errors seen but considered only a warning
my @RsyncWarnings = ();

my $SigIntSeen = 0;

my $RetryBackupIterations = 0;

#
# If a particular restore type needs to have the perms fixed, 
# it should set one of these global flags and then the perms
# will be fixed just before exit.
#
my $RUN_RTI_PERMS = 0;
my $RUN_DAISY_PERMS = 0;

#
# If the user permits it via the "--harden_linux" command line option,
# run the harden_linux.pl script after restoration of some files.
#
my $RUN_HARDEN_LINUX = 0;

my $OS = "";

#
# The command line must be recorded before the GetOptions modules
# is called or any options will be removed.
#
my $COMMAND_LINE = get_command_line();

# command line options
my $HELP = 0;
my $VERSION = 0;
my $CONFIGFILE = $EMPTY_STR;
my $REPORT_CONFIGFILE = 0;
my $GEN_DEF_CONFIGFILE = 0;
my $SEND_SUMMARY = 0;
my @EMAIL_RECIPIENTS = @EMPTY_LIST;
my @PRINTERS = @EMPTY_LIST;
my $FORMAT = 0;
my $INSTALL = 0;
my $UNINSTALL_PRIMARY = 0;
my $INFO_PRIMARY = 0;
my $INSTALL_SECONDARY = 0;
my $UNINSTALL_SECONDARY = 0;
my $INFO_SECONDARY = 0;
my $INSTALL_CLOUD = 0;
my $UNINSTALL_CLOUD = 0;
my $INFO_CLOUD = 0;
my $PRIMARY_SERVER = $EMPTY_STR;
my $MOUNT = 0;
my $UNMOUNT = 0;
my @BACKUP = @EMPTY_LIST;
my @RESTORE = @EMPTY_LIST;
my @RESTORE_EXCLUDES = @EMPTY_LIST;
my @SinglefilesCLO = @EMPTY_LIST;
my $RestoreUpgradeCLO = 0;
my $RootDirCLO = $DEF_ROOTDIR;
my @LIST = @EMPTY_LIST;
my $DAISY =  0;
my $RTI =  0;
my $VERBOSE = 0;
my $FORCE = 0;
my $SHOW_ONLY = 0;
my $LogfileDirCLO = $EMPTY_STR;
my $CLOUD = 0;
my $CLOUD_SERVER = $EMPTY_STR;
my $PERMS_GENERATE = 0;
my $PERMS_UPLOAD = 0;
my $PERMS_DOWNLOAD = 0;
my $PERMS_RESTORE = 0;
my $SERVER = 0;
my $RSYNC_ACCOUNT = $EMPTY_STR;
my $RsyncServer = $EMPTY_STR;
my $RsyncDir = $EMPTY_STR;
my @RSYNC_OPTIONS = @EMPTY_LIST;
my $RSYNC_TRIAL = 0;
my $RSYNC_NICE = 1;
my $RSYNC_METADATA = 1;
my $RSYNC_COMPRESSION = $DEF_RSYNC_COMPRESSION;
my $RSYNC_TIMEOUT = $DEF_RSYNC_TIMEOUT;
my $RUNTIME_CLEANUP = 0;
my $DEVICE = $EMPTY_STR;
my $DEVICE_VENDOR = "WD";
my $DEVICE_MODEL = "My Passport";
my $USB_DEVICE = 0;
my $NetworkDeviceCLO = $DEF_NETWORK_DEVICE;
my $GETINFO = 0;
my $DRY_RUN = 0;
my $HARDEN_LINUX = 1;
my $RetryBackupCLO = $DEF_RETRY_BACKUP;
my $RetryBackupReps = 0;
my $RetryBackupWait = 0;
my $TEST_GET_SHOPCODE = 0;
my $TEST_PROCESS_LOCK = 0;
my $TEST_NOTIFICATION_NO_BACKUP_DEVICE = 0;
my $TEST_GEN_USERS_LISTING_FILE = 0;
my $TEST_GEN_USERS_SHADOW_FILE = 0;
my $TEST_RETRY_BACKUP = 0;
my $TEST_GEN_DEF_CONFIG_FILE = 0;
my $TEST_PARSE_RSYNC_STATS = 0;
my $TEST_PARSE_RSYNC_LOG = $EMPTY_STR;
my $TEST_RSYNC_EXIT_STATUS = 0;
my $TEST_EDIT_AUTH_KEYS = 0;
my $TEST_RESTORE_UPGRADE_ADD_USERS = 0;
my $TEST_RESTORE_UPGRADE_ADJUST_USERS = 0;
my $TEST_RESTORE_UPGRADE_SAMBA_CONF = 0;
my $TEST_RESTORE_UPGRADE_SAMBA_PASSWD = 0;
my $TEST_RESTORE_UPGRADE_HOMEDIR_OWNERS = 0;
my $DEBUGMODE = 0;

GetOptions(
	"help" => \$HELP,
	"version" => \$VERSION,
	"configfile=s" => \$CONFIGFILE,
	"report-configfile" => \$REPORT_CONFIGFILE,
	"gen-default-configfile" => \$GEN_DEF_CONFIGFILE,
	"send-summary" => \$SEND_SUMMARY,
	"email=s" => \@EMAIL_RECIPIENTS,
	"printer=s" => \@PRINTERS,
	"format" => \$FORMAT,
	"install|install-primary" => \$INSTALL,
	"uninstall-primary" => \$UNINSTALL_PRIMARY,
	"info-primary" => \$INFO_PRIMARY,
	"install-secondary" => \$INSTALL_SECONDARY,
	"uninstall-secondary" => \$UNINSTALL_SECONDARY,
	"info-secondary" => \$INFO_SECONDARY,
	"install-cloud" => \$INSTALL_CLOUD,
	"uninstall-cloud" => \$UNINSTALL_CLOUD,
	"info-cloud" => \$INFO_CLOUD,
	"primary-server=s" => \$PRIMARY_SERVER,
	"mount" => \$MOUNT,
	"unmount|umount" => \$UNMOUNT,
	"backup=s" => \@BACKUP,
	"exclude|backup-exclude=s" => \@EXCLUDES,
	"restore=s" => \@RESTORE,
	"restore-exclude=s" => \@RESTORE_EXCLUDES,
	"restore-upgrade" => \$RestoreUpgradeCLO,
	"rootdir=s" => \$RootDirCLO,
	"list=s" => \@LIST,
	"daisy" => \$DAISY,
	"rti" => \$RTI,
	"verbose" => \$VERBOSE,
	"force" => \$FORCE,
	"show-only" => \$SHOW_ONLY,
	"logfiledir=s" => \$LogfileDirCLO,
	"singlefile=s" => \@SinglefilesCLO,
	"cloud" => \$CLOUD,
	"cloud-server=s" => \$CLOUD_SERVER,
	"generate-permfiles" => \$PERMS_GENERATE,
	"upload-permfiles" => \$PERMS_UPLOAD,
	"download-permfiles" => \$PERMS_DOWNLOAD,
	"restore-from-permfiles" => \$PERMS_RESTORE,
	"server" => \$SERVER,
	"rsync-server=s" => \$RsyncServer,
	"rsync-dir=s" => \$RsyncDir,
	"rsync-account=s" => \$RSYNC_ACCOUNT,
	"rsync-trial" => \$RSYNC_TRIAL,
	"rsync-option=s" => \@RSYNC_OPTIONS,
	"rsync-nice!" => \$RSYNC_NICE,
	"rsync-metadata!" => \$RSYNC_METADATA,
	"rsync-compression" => \$RSYNC_COMPRESSION,
	"rsync-timeout=s" => \$RSYNC_TIMEOUT,
	"retry-backup" => \$RetryBackupCLO,
	"retry-reps=s" => \$RetryBackupReps,
	"retry-wait=s" => \$RetryBackupWait,
	"runtime-cleanup" => \$RUNTIME_CLEANUP,
	"device=s" => \$DEVICE,
	"device-vendor=s" => \$DEVICE_VENDOR,
	"device-model=s" => \$DEVICE_MODEL,
	"usb-device" => \$USB_DEVICE,
	"network-device=s" => \$NetworkDeviceCLO,
	"getinfo" => \$GETINFO,
	"dry-run" => \$DRY_RUN,
	"harden-linux!" => \$HARDEN_LINUX,
	"test-get-shopcode" => \$TEST_GET_SHOPCODE,
	"test-process-lock" => \$TEST_PROCESS_LOCK,
	"test-notify-no-backup-device" => \$TEST_NOTIFICATION_NO_BACKUP_DEVICE,
	"test-gen-users-listing-file" => \$TEST_GEN_USERS_LISTING_FILE,
	"test-gen-users-shadow-file" => \$TEST_GEN_USERS_SHADOW_FILE,
	"test-retry-backup" => \$TEST_RETRY_BACKUP,
	"test-gen-default-config-file" => \$TEST_GEN_DEF_CONFIG_FILE,
	"test-parse-rsync-stats" => \$TEST_PARSE_RSYNC_STATS,
	"test-parse-rsync-log=s" => \$TEST_PARSE_RSYNC_LOG,
	"test-rsync-exit-status" => \$TEST_RSYNC_EXIT_STATUS,
	"test-edit-auth-keys" => \$TEST_EDIT_AUTH_KEYS,
	"test-restore-upgrade-add-users" => \$TEST_RESTORE_UPGRADE_ADD_USERS,
	"test-restore-upgrade-adjust-users" => \$TEST_RESTORE_UPGRADE_ADJUST_USERS,
	"test-restore-upgrade-samba-conf" => \$TEST_RESTORE_UPGRADE_SAMBA_CONF,
	"test-restore-upgrade-samba-passwd" => \$TEST_RESTORE_UPGRADE_SAMBA_PASSWD,
	"test-restore-upgrade-homedir-owners" => \$TEST_RESTORE_UPGRADE_HOMEDIR_OWNERS,
	"debugmode" => \$DEBUGMODE,
) || die "Error: invalid command line option, exiting...\n";


# some options automatically imply --server
if ($INSTALL || $UNINSTALL_PRIMARY || $INFO_PRIMARY ||
    $UNINSTALL_SECONDARY || $INSTALL_SECONDARY || $INFO_SECONDARY) {
    $SERVER = 1;
}

# some other options automatically imply --cloud
if ($INSTALL_CLOUD || $UNINSTALL_CLOUD || $INFO_CLOUD ||
    $PERMS_GENERATE || $PERMS_UPLOAD || $PERMS_DOWNLOAD || $PERMS_RESTORE) {
    $CLOUD = 1;
}


# check command line for obviously inconsistent command line options
# and rule them out now
unless (tfr_is_cmd_line_consistent()) {
    exit($EXIT_COMMAND_LINE);
}


# --version
if ($VERSION != 0) {
    print("OSTools Version: __OSTOOLS_VERSION__\n");
    print("$CVS_REVISION\n");
    exit($EXIT_OK);
}

# --help
if ($HELP != 0) {
    usage();
    exit($EXIT_OK);
}


if ($EUID != 0) {
    print {*STDERR} "$0 must be run as root or with sudo\n";
    exit($EXIT_MUST_BE_ROOT);
}


#
# decide the application platform and set the relevant
# global vars appropriately.
#
unless (tfr_pick_rti_or_daisy()) {
    exit($EXIT_COMMAND_LINE);
}


#
# initialize log file configuration
#
unless (tfr_log_setup($LogfileDirCLO)) {
    print {*STDERR} "could not initialize log file configuration\n";
    exit($EXIT_LOGFILE_SETUP);
}


# --debug
# output extra messages if debug requested, start off
# with some info sent to debug log file
if ($DEBUGMODE) {
    debuglog("starting $PROGNAME $CVS_REVISION");
    debuglog("invoked with command line:");
    debuglog("$COMMAND_LINE");
    debuglog("CURRENT LOGFILE = <$LOGFILE>");
    debuglog("SUMMARY LOGFILE = <$SummaryLogfile>");
}

#
# convert all the command line options that are
# comma separated strings into lists.
#
my @list_cmd_line_options = (
    \@BACKUP,
    \@EXCLUDES,
    \@EMAIL_RECIPIENTS,
    \@PRINTERS,
    \@RESTORE,
    \@RESTORE_EXCLUDES,
    \@LIST,
    \@SinglefilesCLO,
    \@RSYNC_OPTIONS,
);
for my $ref (@list_cmd_line_options) {
    if (scalar(@{$ref})) {
	@{$ref} = split(',', join(',', @{$ref}));
    }
}


#
# Check all the options that allow lists for insecure values.
#
my @options_allowing_lists = (
    ['@BACKUP', @BACKUP],
    ['@EXCLUDES', @EXCLUDES],
    ['@EMAIL_RECIPIENTS', @EMAIL_RECIPIENTS],
    ['@PRINTERS', @PRINTERS],
    ['@RESTORE', @RESTORE],
    ['@RESTORE_EXCLUDES', @RESTORE_EXCLUDES],
    ['@LIST', @LIST],
    ['@SinglefilesCLO', @SinglefilesCLO],
    ['@RSYNC_OPTIONS', @RSYNC_OPTIONS],
);

# loop through each anonymous list
for my $rec_nr (0 .. $#options_allowing_lists) {

    # each anonymous list has a least one element, the name
    my $rec_name = $options_allowing_lists[$rec_nr][0];

    # if the maximum index nr of anonymous list is 0, there is only
    # the name field
    my $max_rec_idx = $#{$options_allowing_lists[$rec_nr]};

    # loop through any elements present after the name
    for my $elem_nr (1 .. $max_rec_idx) {
	my $elem_val = $options_allowing_lists[$rec_nr][$elem_nr];
	if (is_input_insecure($elem_val)) {
	    showerror("insecure value contained in $rec_name: $elem_val");
	    shutdown_and_exit($EXIT_COMMAND_LINE);
	}
    }
}

#
# Check the values of options that allow input from user.
#
my @options_allowing_vals = (
    ['$DEVICE', $DEVICE],
    ['$DEVICE_VENDOR', $DEVICE_VENDOR],
    ['$DEVICE_MODEL', $DEVICE_MODEL],
    ['$RsyncServer', $RsyncServer],
    ['$RsyncDir', $RsyncDir],
    ['$RSYNC_ACCOUNT', $RSYNC_ACCOUNT],
    ['$RSYNC_TIMEOUT', $RSYNC_TIMEOUT],
    ['$RootDirCLO', $RootDirCLO],
    ['$PRIMARY_SERVER', $PRIMARY_SERVER],
    ['$NetworkDeviceCLO', $NetworkDeviceCLO],
    ['$RetryBackupReps', $RetryBackupReps],
    ['$RetryBackupWait', $RetryBackupWait],
);

# loop through each anonymous list
for my $rec_nr (0 .. $#options_allowing_vals) {

    # each anonymous list has a least one element, the name
    my $rec_name = $options_allowing_vals[$rec_nr][0];

    # get value of 2nd element
    my $rec_val = $options_allowing_vals[$rec_nr][1];

    # $NetworkDeviceCLO is not allowed to be empty
    if ($rec_name eq '$NetworkDeviceCLO' && $rec_val eq "") {
	showerror("network interface device name is not allowed to be empty");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }

    # test value, empty string is considered OK... if not,
    # test before this code.
    if (is_input_insecure($rec_val)) {
	showerror("insecure value contained in $rec_name: $rec_val");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }
}

# verify network interface is valid
if (verify_network_device($NetworkDeviceCLO)) {
    loginfo("network interface device name verified: $NetworkDeviceCLO");
}
else {
    showerror("network interface device name does not verify: $NetworkDeviceCLO");
    shutdown_and_exit($EXIT_COMMAND_LINE);
}


$OS = plat_os_version();
if ($OS eq "") {
	logerror("Unknown operating system.");
	shutdown_and_exit($EXIT_PLATFORM);
}


# --rootdir=/some/path
unless (-d $RootDirCLO) {
	logerror("The directory specified with --rootdir not found: $RootDirCLO");
	shutdown_and_exit($EXIT_ROOTDIR);
}


# Where do our "OSTOOLS" typically reside?
my @ostools_dir_paths = (
    "/teleflora/ostools",
    "/usr2/ostools",
    "/d/ostools"
);

foreach (@ostools_dir_paths) {
    if (-e $_) {
	$ToolsDir = $_;
	last;
    }
}

unless ($ToolsDir) {
    logerror("OSTools directory does not exist at: @ostools_dir_paths");
    shutdown_and_exit($EXIT_TOOLSDIR);
}

#
# now that the tools directory location is set, the other locations
# which depend on location of tools can be set:
#   1) path to tools config dir
#   2) path to our default config file
#   3) path to our default config dir
#   4) path to sudoers config dir
#   5) path to sudoers content config file
#
my $TOOLS_CONFIG_DIR_PATH      = $ToolsDir . '/' . $DEF_TOOLS_CONFIG_DIR_NAME;
my $POS_CONFIG_FILE_PATH       = $TOOLS_CONFIG_DIR_PATH . '/' . $DEF_CONFIG_FILENAME;
my $POS_CONFIG_DIR_PATH        = $TOOLS_CONFIG_DIR_PATH . '/' . $DEF_CONFIG_DIR_NAME;

my $SUDOERS_CONFIG_DIR_PATH    = $TOOLS_CONFIG_DIR_PATH . '/' . 'sudoers.d';
my $SUDOERS_CONTENT_FILE_PATH  = $SUDOERS_CONFIG_DIR_PATH . '/'. $DEF_SUDOERS_CONTENT_FILE;


#
# there are some files and directories in "/usr2" that must
# not be backed up in place.  Entities on this list will be
# skipped when backing up to an rsync server.
#
my @USR2_EXCLUSIONS  = (
    # constraint            item to be excluded
    [ $DEVTYPE_CLOUD,       $RTI_DELVCONF],
    [ $DEVTYPE_SERVER,      $RTI_DOVE_CMD],
    [ $DEVTYPE_SERVER,      $DEF_RTI_RSYNC_BU_DIR],
    [ $DEVTYPE_SERVER,      $ToolsDir],
    [ $DEVTYPE_ANY,         $RTI_BASIS_CACHE_DIR],
);


####################################
###                              ###
###          MAINLINE            ###
###                              ###
####################################


loginfo("");
loginfo("|===========================================");
loginfo("| BEGIN MAINLINE: $PROGNAME $CVS_REVISION");
loginfo("|===========================================");
loginfo("|");
loginfo("| invoked with command line:");
loginfo("| $COMMAND_LINE");
loginfo("|");
loginfo("|===========================================");


####################################
###                              ###
### PROCESS COMMAND LINE OPTIONS ###
###                              ###
####################################

#
# if there were any command line options specifying test functions,
# the script will exit after running the test.
#
tfr_test_functions();


#
# the rsync dir can be specified either:
#   1) as a command line option
#   2) as a command line argument
#   3) in the config file
#
# if a path is specified, but the rsync server is not, then
# do a rsync to a local file system.
#
if (defined($ARGV[0])) {
    $RsyncDir = $ARGV[0];
    if (is_input_insecure($RsyncDir)) {
	print("insecure value for path specified with --rsync-dir: $RsyncDir\n");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }
}

#
# --backup=<backup_type_list>
#
# Check backup type list
#
# Check the backup types specified on the command line and
# let the user know if there is an unknown or unsupported
# backup type specified.
#
if ($#BACKUP >= 0) {
    foreach my $backup_type (@BACKUP) {

	next if (grep {/$backup_type/} @UNIQUE_BACKUP_TYPES);

	next if (grep {/$backup_type/} @BACKUP_TYPES);

	showerror("unknown backup type specified: $backup_type");
	shutdown_and_exit($EXIT_BACKUP_TYPE);
    }
}


#
# --restore=<restore_type_list>
#
# Check restore type list
#
# Check the restore types specified on the command line and
# let the user know if there is an error.
#
if ($#RESTORE >= 0) {
    foreach my $restore_type (@RESTORE) {
	unless (grep {/^$restore_type$/} @RESTORE_TYPES) {
	    showerror("unknown restore type specified: $restore_type");
	    shutdown_and_exit($EXIT_RESTORE);
	}
    }
}


# what is the hostname?
my $Hostname = hostname();
loginfo("hostname: $Hostname");

# what is the shopcode?
my $Shopcode = get_shopcode();
loginfo("shopcode: $Shopcode");

#
# --server
#
if ($SERVER) {
    $DeviceType = $DEVTYPE_SERVER;
    unless ($RsyncServer) {
	$RsyncServer = $DEF_RSYNC_SERVER;
	loginfo("using default backup server: $RsyncServer");
    }
    unless ($RSYNC_ACCOUNT) {
	$RSYNC_ACCOUNT = $DEF_RSYNC_ACCOUNT;
	loginfo("using default server account name: $RSYNC_ACCOUNT");
    }
}

#
# --cloud
#
if ($CLOUD) {
    $DeviceType = $DEVTYPE_CLOUD;
    unless ($CLOUD_SERVER) {
	$CLOUD_SERVER = $DEF_CLOUD_SERVER;
	loginfo("using default cloud server: $CLOUD_SERVER");
    }
    if ($RSYNC_ACCOUNT) {
	if ($RSYNC_ACCOUNT eq $DEF_RSYNC_ACCOUNT) {
	    showerror("error: the cloud account name can not be: $DEF_RSYNC_ACCOUNT");
	    shutdown_and_exit($EXIT_CLOUD_ACCOUNT_NAME);
	}
	elsif ($RSYNC_ACCOUNT !~ m/[[:alpha:]]{1}\w*-\d{8}/) {
	    showerror("error: the cloud account name must be: <name> + '-' + <shopcode>");
	    shutdown_and_exit($EXIT_CLOUD_ACCOUNT_NAME);
	}
    }
    else {
	$RSYNC_ACCOUNT = tfr_cloud_account_name();
	if ($RSYNC_ACCOUNT) {
	    loginfo("using cloud account name: $RSYNC_ACCOUNT");
	}
	else {
	    showerror("could not determine cloud account name");
	    shutdown_and_exit($EXIT_CLOUD_ACCOUNT_NAME);
	}
    }
}
else {
    if ($RestoreUpgradeCLO) {
	showerror("restore upgrade only allowed from cloud server");
	shutdown_and_exit($EXIT_CLOUD_ACCOUNT_NAME);
    }
}

#
# --device
#
if ($DEVICE) {
    $DeviceType = $DEVTYPE_DEVICE;
}


#
# if the config file path has a non-empty value then
# it has already been set on the command line so
# check it for security issues.
#
# else if not set, then set it to the default config file path.
#
if ($CONFIGFILE) {
    if (is_input_insecure($CONFIGFILE)) {
	showerror("insecure value for path to config file: $CONFIGFILE");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }
}
else {
    $CONFIGFILE = tfrm_pathto_def_pos_config_file();
}


#
# --uninstall-primary
#
if ($UNINSTALL_PRIMARY) {
    shutdown_and_exit(tfr_uninstall_primary($RSYNC_ACCOUNT, $DeviceType));
}

#
# --install
# --install-primary
#
if ($INSTALL) {
    shutdown_and_exit(tfr_install_primary($RSYNC_ACCOUNT, $DeviceType));
}

#
# --info-primary
# report info about primary configuration and exit
#
if ($INFO_PRIMARY) {
    shutdown_and_exit(tfr_info_primary($RSYNC_ACCOUNT));
}

#
# --uninstall-secondary
#
if ($UNINSTALL_SECONDARY) {
    shutdown_and_exit(tfr_uninstall_secondary($RSYNC_ACCOUNT));
}

#
# --install-secondary
#
if ($INSTALL_SECONDARY) {
    unless ($PRIMARY_SERVER) {
	$PRIMARY_SERVER = $DEF_PRIMARY_SERVER;
	loginfo("using default primary server: $PRIMARY_SERVER");
    }

    shutdown_and_exit(tfr_install_secondary($RSYNC_ACCOUNT, $PRIMARY_SERVER));
}

#
# --info-secondary
# report info about secondary configuration and exit
#
if ($INFO_SECONDARY) {
    shutdown_and_exit(tfr_info_secondary($RSYNC_ACCOUNT));
}

#
# --uninstall-cloud
#
if ($UNINSTALL_CLOUD) {
    shutdown_and_exit(tfr_uninstall_primary($RSYNC_ACCOUNT, $DeviceType));
}

#
# --install-cloud
#
if ($INSTALL_CLOUD) {
    my $rc = tfr_install_primary($RSYNC_ACCOUNT, $DeviceType);
    if ($rc == $EXIT_OK) {
	my @platform_bu_types = tfr_get_platform_bu_types(@BACKUP_TYPES);
	$rc = tfr_generate_perm_files(@platform_bu_types);
    }

    shutdown_and_exit($rc);
}

#
# --info-cloud
# report info about cloud configuration and exit
#
if ($INFO_CLOUD) {
    shutdown_and_exit(tfr_info_cloud($RSYNC_ACCOUNT));
}

#
# --generate-permfiles
#
# generate perm files for all backup types
#
if ($PERMS_GENERATE) {
    my @platform_bu_types = tfr_get_platform_bu_types(@BACKUP_TYPES);
    shutdown_and_exit(tfr_generate_perm_files(@platform_bu_types));
}

#
# --upload-permfiles
#
# upload set of existing perm files to the cloud server
#
if ($PERMS_UPLOAD) {
    shutdown_and_exit(tfr_upload_perm_files($RSYNC_ACCOUNT, $CLOUD_SERVER));
}

#
# --download-permfiles
#
# download existing set of perm files from the cloud server.
#
if ($PERMS_DOWNLOAD) {
    my @platform_bu_types = tfr_get_platform_bu_types(@BACKUP_TYPES);
    shutdown_and_exit(tfr_download_perm_files($RSYNC_ACCOUNT, $CLOUD_SERVER, @platform_bu_types));
}

#
# --restore-from-permfiles
#
# restore the perms from a set of perm files
#
if ($PERMS_RESTORE) {
    my @platform_bu_types = tfr_get_platform_bu_types(@BACKUP_TYPES);
    shutdown_and_exit(tfr_restore_from_perm_files($RSYNC_ACCOUNT, $CLOUD_SERVER, @platform_bu_types));
}


#
# "it was decided" that the config file values takes precedence
# over command line values.  Thus, this is the point that the
# config file can be read.
#
# read the configuration file(s)
#
my $ConfigFilePath = tfrm_pathto_config_file();
my $ConfigFileCount = tfr_read_configuration($ConfigFilePath);
showinfo("number of conf files read: $ConfigFileCount") if ($VERBOSE);


# --report-configfile
# report only, then exit
if ($REPORT_CONFIGFILE) {
    shutdown_and_exit($EXIT_OK);
}

# --gen-configfile
# generate and install a default config file
if ($GEN_DEF_CONFIGFILE) {
    my $config_file_path = tfrm_pathto_def_pos_config_file();
    my $rc = $EXIT_OK;

    my $installed_config_file_path = tfr_install_default_config_file($config_file_path);
    if ($installed_config_file_path) {
	showinfo("default config file generated: $installed_config_file_path");
    }
    else {
	showerror("could not generate default config file: $config_file_path");
	$rc = $EXIT_DEF_CONFIG_FILE;
    }

    shutdown_and_exit($rc);
}

#
# if "--retry-backup" is specified, then check the range of values for
# "--retry-reps" and "--retry-wait".  If their values are 0, set them
# to default values and log values.
#
if ($RetryBackupCLO) {
    if ( ($RetryBackupReps < 0) || ($RetryBackupReps > $MAX_RETRY_BACKUP_REPS) ) {
	showerror("backup retry reps must be >= 0 and <= $MAX_RETRY_BACKUP_REPS: $RetryBackupReps");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }
    if ( ($RetryBackupWait < 0) || ($RetryBackupWait > $MAX_RETRY_BACKUP_WAIT) ) {
	showerror("backup retry wait must be >= 0 and <= $MAX_RETRY_BACKUP_WAIT: $RetryBackupWait");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }

    $RetryBackupReps = $DEF_RETRY_BACKUP_REPS if ($RetryBackupReps == 0);
    loginfo("backup retries set to: $RetryBackupReps");

    $RetryBackupWait = $DEF_RETRY_BACKUP_WAIT if ($RetryBackupWait == 0);
    loginfo("backup wait set to: $RetryBackupWait");
}


#
# if not doing a cloud backup or a backup server and
# a rsync destination path IS specified, it's a copy
# to a local file system so make sure that the
# destination exists.
#
if (!$CLOUD && !$RsyncServer) {
    if ($RsyncDir) {
	if (-d $RsyncDir) {
	    $DeviceType = $DEVTYPE_FILE_SYSTEM;
	    loginfo("backup to local filesystem: $RsyncDir");
	}
	else {
	    showerror("Destination directory must exist");
	    shutdown_and_exit($EXIT_COMMAND_LINE);
	}
    }
}

#
# Consistency check for backup device conflicts.
#
if ($DEVICE || $USB_DEVICE) {
    my $device_str = ($DEVICE) ? "--device" : "--usb-device";
    if ($RsyncServer) {
	showerror("Can not specify both $device_str and --rsync-server");
	shutdown_and_exit($EXIT_BACKUP_DEVICE_CONFLICT);
    }
    elsif ($RsyncDir) {
	showerror("Can not specify both $device_str and rsync destination path");
	shutdown_and_exit($EXIT_BACKUP_DEVICE_CONFLICT);
    }
}


#
# If specifed, either find and verify a USB device, or exit.
#
# --usb-device
#
if ($USB_DEVICE) {
    if ($OS eq "RHEL4") {
	showerror("USB devices other than WD Passports are not supported on $OS");
	shutdown_and_exit($EXIT_USB_DEVICE_UNSUPPORTED);
    }
    else {
	$DEVICE = find_usb_device();
	if ($DEVICE) {
	    if (device_is_verified($DEVICE)) {
		$DeviceType = $DEVTYPE_USB;
	    }
	    else {
		showerror("Can't verify USB device: $DEVICE");
		shutdown_and_exit($EXIT_DEVICE_VERIFY);
	    }
	}
	else {
	    showerror("Can't find USB backup device");
	    shutdown_and_exit($EXIT_USB_DEVICE_NOT_FOUND);
	}
    }
}


#
# If the backup device is an image file, make sure that it exists and
# is at least $DEVICE_IMAGE_SIZE in size - arbitrary yes, but at least
# some measure of defense.
#
# --device
#
if ($DEVICE) {
    if ($DEVICE eq $DEVTYPE_SHOW_ONLY) {
	$DeviceType = $DEVTYPE_SHOW_ONLY;
    }
    elsif (-f $DEVICE) {
	$DeviceType = $DEVTYPE_IMAGE;
    }
    elsif (-b $DEVICE) {
	$DeviceType = $DEVTYPE_BLOCK unless ($DeviceType);
    }
    else {
	unless (device_is_verified($DEVICE)) {
	    logerror("Backup device verification error for: $DEVICE");
	    shutdown_and_exit($EXIT_DEVICE_VERIFY);
	}
    }
}

#
# If at this point the user hasn't specified an rsync to a
# backup server or to a local file system or to the cloud or
# explicity to a backup device, then look for a WD Passport.
#
else {
    unless ($SERVER || $RsyncDir || $CLOUD) {
	$DEVICE = find_passport();
    }
}


#
# --runtime-cleanup
#
# cleanup the process lock file and the ssh tunnel socket.
#
if ($RUNTIME_CLEANUP) {
    shutdown_and_exit(tfr_runtime_cleanup());
}


#
# At this point, assert there is some backup device or
# it's an error.
#
if ($SERVER) {
    showinfo("using backup server: $RsyncServer");
}
elsif ($RsyncDir) {
    showinfo("using local file system at path: $RsyncDir");
}
elsif ($DeviceType eq $DEVTYPE_SHOW_ONLY) {
    showinfo("using special backup device: $DEVICE");
}
elsif (-f $DEVICE) {
    showinfo("using backup image file: $DEVICE");
}
elsif (-b $DEVICE) {
    showinfo("using backup device: $DEVICE");
}
elsif ($CLOUD) {
    showinfo("using cloud backup");
}
else {
    # If at this point the device is still unknown, and there are
    # one or more backup sets specified, then send notifications.
    showerror("no backup device found.");
    if ($#BACKUP >= 0) {
	tfr_notification_no_backup_device();
    }
    shutdown_and_exit($EXIT_BACKUP_DEVICE_NOT_FOUND);
}


# --getinfo
if ($GETINFO) {
    if ($RsyncServer || ($RsyncServer eq "" && $RsyncDir)) {
	showinfo("The --getinfo option does not apply to rsync servers or file systems");
    }
    else {
	get_backup_device_info($DEVICE, $MOUNTPOINT);
    }

    shutdown_and_exit($EXIT_OK);
}


# --mount
if ($MOUNT) {
    my $rc = $EXIT_OK;
    if ($DEVICE) {
	mount_device("rw");
    }
    else {
	showerror("backup device not specified");
	$rc = $EXIT_DEVICE_NOT_SPECIFIED;
    }

    shutdown_and_exit($rc);
}


# --unmount
# --umount
if ($UNMOUNT) {
    my $rc = $EXIT_OK;
    if ($DEVICE) {
	unmount_device();
    }
    else {
	showerror("backup device not specified");
	$rc = $EXIT_DEVICE_NOT_SPECIFIED;
    }

    shutdown_and_exit($rc);
}


# --format
if ($FORMAT) {
    # unless "--force" is also specified, verify format with "yes/no"
    unless ($FORCE) {
	my $answer = get_yn("Format backup device: $DEVICE");
	unless ($answer) {
	    shutdown_and_exit($EXIT_OK);
	}
    }

    my $rc = tfr_device_format_ext2($DEVICE, $DeviceType);
    if ($rc == 0) {
	showinfo("device formatted: $DEVICE");
	$rc = $EXIT_OK;
    }
    else {
	showerror("Format failed: $DEVICE");
	$rc = $EXIT_FORMAT;
    }

    shutdown_and_exit($rc);
}


#=======================================================================
# setup the interrupt signal handler
# ----------------------------------
#
# catch interrupt signal in order to exit cleanly.
#
#=======================================================================
tfr_set_sigint_handler();


# --list
if ($#LIST >= 0) {

    if ($CLOUD) {
	showerror("the --cloud option may not be used with --list option");
	shutdown_and_exit($EXIT_COMMAND_LINE);
    }

    #
    # Check the backup types to list that were specified on the command line and
    # let the user know if there is an error and what the supported backup types are.
    #
    for my $backup_type (@LIST) {
	unless (grep {/^$backup_type$/} @BACKUP_TYPES) {
	    logerror("Error: backup type specified: $backup_type");
	    logerror("Error: supported backup types: @BACKUP_TYPES");
	    shutdown_and_exit($EXIT_LIST_UNSUP);
	}
    }
    if (tfr_list_files(\@LIST, $DeviceType) == 0) {
	shutdown_and_exit($EXIT_OK);
    }
    else {
	logerror("error while listing files");
	shutdown_and_exit($EXIT_LIST);
    }
}

#=======================================================================
# attempt to obtain a process lock
# --------------------------------
#
# If a process lock can not be obtained, will not return, will just
# clean up and exit.
#
# If "--dry-run" specified, will return without getting process lock.
#
#=======================================================================
tfr_process_lock_obtain();


# --backup
# Make sure this block of code comes *after* the "mount",
# "unmount" and "format" blocks of code.
if ($#BACKUP >= 0) {
	if ($DEBUGMODE) {
	    push(@BACKUP, "specialfiles");
	}

	tfr_init_rsync_stats();

	tfr_rsync_status_warnings_init();

	tfr_backup_init_default_excludes();

	my $begin_time = time;

	my $backup_returnval = tfr_backup_files(\@BACKUP, \@EXCLUDES);

	my $end_time = time;

	if ($DEBUGMODE) {
	    loginfo("backup return value: $backup_returnval");
	}

	if (! defined($ExitTable{$backup_returnval})) {
	    if ($DEBUGMODE) {
		loginfo("undefined backup return value: $backup_returnval");
	    }
	    $backup_returnval = $EXIT_UNKNOWN;
	}

	# send out an error notification if needed
	unless ($backup_returnval == $EXIT_OK) {
	    tfr_notification_backup_error($backup_returnval);
	}

	# process the backup summary report
	tfr_backup_summary_report($begin_time, $end_time, $backup_returnval);

	# always try to backup the POS summary log
	if (tfr_backup_pos_summary_log()) {
	    loginfo("POS summary log file resynced: $SummaryLogfile");
	}
	else {
	    logerror("could not resync POS summary log: $SummaryLogfile");
	}

	if (tfr_process_lock_release()) {
	    loginfo("process lock released");
	}

	$ExitStatus = $backup_returnval;
}


# --restore
if ($#RESTORE >= 0) {
	# if RTI and restoring "in place", then exclude these:
	if ($RTI && is_restore_in_place()) {
	    push(@RESTORE_EXCLUDES, @DEF_RTI_RESTORE_EXCLUDES);
	}

	my $rc = tfr_restore_files(\@RESTORE, \@RESTORE_EXCLUDES, $RootDirCLO);
	if ($rc == 0) {
	    if ($RUN_RTI_PERMS) {
		set_rti_perms();
	    }
	    if ($RUN_DAISY_PERMS) {
		set_daisy_perms();
	    }
	    if ($HARDEN_LINUX && $RUN_HARDEN_LINUX) {
		run_harden_linux();
	    }
	}
	else {
	    print STDERR "the --restore option reported errors - please check log file\n";
	    $ExitStatus = $EXIT_RESTORE;
	}

	if (tfr_process_lock_release()) {
	    loginfo("process lock released");
	}
}

loginfo("END MAINLINE: $PROGNAME");

unless (tfr_log_conclude()) {
    print {*STDERR} "could not conclude log file operations\n";
}


exit($ExitStatus);

#####################################################################
#####################################################################
#####################################################################


sub usage
{
    print("$PROGNAME $CVS_REVISION\n");
    print("\n");
    print("For full documenation, enter the command:  perldoc $0\n");
    print("\n");

    print("SYNOPSIS\n");
    print("$PROGNAME --help\n");
    print("$PROGNAME --version\n");
    print("\n");

    print("$PROGNAME --install-primary (--server | --cloud) [--rsync-account=name]\n");
    print("$PROGNAME --uninstall-primary [--rsync-account=name]\n");
    print("$PROGNAME --info-primary [--rsync-account=name]\n");
    print("$PROGNAME --install-secondary [--primary-server=addr] [--rsync-account=name]\n");
    print("$PROGNAME --uninstall-secondary [--rsync-account=name]\n");
    print("$PROGNAME --info-secondary [--rsync-account=name]\n");
    print("$PROGNAME --install-cloud [--cloud-server=addr] [---rsync-account=name]\n");
    print("$PROGNAME --uninstall-cloud [--cloud-server=addr] [---rsync-account=name]\n");
    print("$PROGNAME --generate-permfiles\n");
    print("$PROGNAME --upload-permfiles [--cloud-server=addr] [--rsync-account=name]\n");
    print("$PROGNAME --download-permfiles [--cloud-server=addr] [--rsync-account=name]\n");
    print("$PROGNAME --restore-from-permfiles [--cloud-server=addr] [--rsync-account=name]\n");
    print("$PROGNAME --backup=<list> --cloud [--cloud-server=addr] [--rsync-account=name]\n");
    print("$PROGNAME --backup=<list> --server [--rsync-server=addr] [--rsync-account=name]\n");
    print("\twhere <list> is one or more of the following comma separated types:\n");
    my @backup_types = @BACKUP_TYPES;
    push(@backup_types, @UNIQUE_BACKUP_TYPES);
    tfr_print_array(\@backup_types, "\t", ",", 6);
    print("$PROGNAME --restore=<list> --cloud [--cloud-server=addr] [--rsync-account=name]\n");
    print("$PROGNAME --restore=<list> --server [--rsync-server=addr] [--rsync-account=name]\n");
    print("\twhere <list> is one or more of the following comma separated types:\n");
    tfr_print_array(\@RESTORE_TYPES, "\t", ",", 6);
    print("$PROGNAME --list=<list>\n");
    print("\twhere <list> is one or more of the following comma separated types:\n");
    tfr_print_array(\@backup_types, "\t", ",", 6);
    print("\n");

    print("$PROGNAME --mount [--device]\n");
    print("$PROGNAME --unmount [--device]\n");
    print("$PROGNAME --getinfo\n");
    print("$PROGNAME --report-configfile\n");
    print("$PROGNAME --gen-configfile\n");
    print("$PROGNAME --send-summary\n");
    print("$PROGNAME --format [--force]\n");
    print("$PROGNAME --runtime-cleanup (--server | --cloud)\n");
    print("\n");

    print("DEFAULTS\n");
    print("The default primary server address: $DEF_PRIMARY_SERVER\n");
    print("The default server account name: $DEF_RSYNC_ACCOUNT\n");
    print("The default cloud server name: $DEF_CLOUD_SERVER\n");
    print("The default cloud account name: 'tfrsync-nnnnnnnn' where nnnnnnnn = shopcode\n");
    print("\n");

    print("OPTIONS for '--backup'\n");
    print("--cloud\n");
    print("--cloud-server=(FQDN | ipaddr)\n");
    print("--server\n");
    print("--rsync-server=(hostname | ipaddr)\n");
    print("--rsync-account=name\n");
    print("--rsync-dir=path\n");
    print("--rsync-option=string\n");
    print("--rsync-timeout=secs\n");
    print("--[no]rsync-nice\n");
    print("--rsync-trial\n");
    print("--rsync-compression\n");
    print("--retry-backup\n");
    print("--retry-reps=number\n");
    print("--retry-wait=seconds\n");
    print("--network-device=s\n");
    print("--singlefile=<list>      where <list> is one or more comma separated paths\n");
    print("--backup-exclude=<list>  where <list> is one or more comma separated paths\n");
    print("--email=<list>           where <list> is one or more comma separated email addresses\n");
    print("--printer=<list>         where <list> is 1 or more comma separated printer names\n");
    print("--rootdir=<path>         where <path> is the path to an existing directory\n");
    print("--show-only              only show the filenames to backup, do not perform back up\n");
    print("\n");

    print("DEFAULTS\n");
    print("The default rsync timeout: $DEF_RSYNC_TIMEOUT seconds, 0 means no timeout\n");
    print("The default rsync compression: " . (($DEF_RSYNC_COMPRESSION) ? "yes" : "no") . "\n");
    print("The default backup retry policy: " . (($DEF_RETRY_BACKUP) ? "do retries" : "no retries") . "\n");
    print("The default number of backup retries: $DEF_RETRY_BACKUP_REPS\n");
    print("The max value for number of backup retries: $MAX_RETRY_BACKUP_REPS\n");
    print("The default backup retry wait (seconds): $DEF_RETRY_BACKUP_WAIT\n");
    print("The max value for backup retry wait (seconds): $MAX_RETRY_BACKUP_WAIT\n");
    print("The default number of ssh open retries: $DEF_SSH_TUNNEL_RETRIES\n");
    print("The default network interface device name: $DEF_NETWORK_DEVICE\n");
    print("The default backup excludes:\n");
    tfr_print_array(\@DEF_BACKUP_EXCLUDES, "\t", ",", 5);
    print("The default RTI backup excludes:\n");
    tfr_print_array(\@DEF_RTI_BACKUP_EXCLUDES, "\t", ",", 5);
    print("The default RTI usr2 backup excludes for backup class \"cloud\":\n");
    print "\t$RTI_DELVCONF\n";
    print("The default RTI usr2 backup excludes for backup class \"server\":\n");
    print "\t$RTI_DOVE_CMD, $DEF_RTI_RSYNC_BU_DIR, $RTI_TOOLSDIR\n";
    print("The default Daisy backup excludes:\n");
    tfr_print_array(\@DEF_DAISY_BACKUP_EXCLUDES, "\t", ",", 5);
    print("\n");

    print("OPTIONS for '--restore'\n");
    print("--restore-exclude=<list>    where <list> is one or more comma separated paths\n");
    print("--singlefile=<list>         where <list> is one or more comma separated paths\n");
    print("--noharden-linux            do not run harden_linux.pl after restore\n");
    print("\n");

    print("DEFAULTS\n");
    print("The default RTI restore excludes:\n");
    tfr_print_array(\@DEF_RTI_RESTORE_EXCLUDES, "\t", ",", 5);
    print("The default number of ssh open retries: $DEF_SSH_TUNNEL_RETRIES\n");
    print("\n");

    print("OPTIONS for any command\n");
    print("--daisy\n");
    print("--rti\n");
    print("--configfile=path\n");
    print("--logfiledir=path\n");
    print("--device=/dev/xxxx where /dev/xxxx is a block device\n");
    print("--device=/path/to/imagefile.img where imagefile.img is prepped image file\n");
    print("--usb-device\n");
    print("--dry-run\n");
    print("\n");

    print("DEFAULTS\n");
    print("The default RTI config file path: $DEF_RTI_CONFIG_FILE_PATH\n");
    print("The default Daisy config file path: $DEF_DAISY_CONFIG_FILE_PATH\n");
    print("The default RTI log file directory: $DEF_RTI_LOG_DIR\n");
    print("The default Daisy log file directory: $DEF_DAISY_LOG_DIR\n");

    return(1);
}


sub tfr_is_cmd_line_consistent
{
    if ($CLOUD && $SERVER) {
	print {*STDERR} "--cloud and --server are mutually exclusive\n";
	return(0);
    }

    #
    # Consistency check for backup device conflicts.
    #
    if ($DEVICE || $USB_DEVICE) {
    
	if ($DEVICE && $USB_DEVICE) {
	    print {*STDERR} "--device=path and --usb-device are mutually exclusive\n";
	    return(0);
	}

	my $device_str = ($DEVICE) ? "--device=path" : "--usb-device";
	if ($RsyncServer) {
	    print {*STDERR} "$device_str and --rsync-server are mutually exclusive\n";
	    return(0);
	}
	elsif ($RsyncDir) {
	    print {*STDERR} "$device_str and --rsync-dir are mutually exclusive\n";
	    return(0);
	}
    }

    return(1);
}


#
# decide if a RTI system or a Daisy system
#
# handle four cases:
# 1. both --daisy and --rti specified
# 2. neither specified
# 3. only --rti specified
# 4. only --daisy rti specified
#
sub tfr_pick_rti_or_daisy
{
    # 1. if both specified, log warning and use "rti".
    # Note that these options are mutally exclusive
    if ( ($RTI == 1) && ($DAISY == 1) ) {
	print {*STDERR} "the --rti and --daisy options are mutally exclusive";
	print {*STDERR} "continuing as if only --rti was specified";
	$DAISY = 0;
    }

    # 2. if neither specified, give preference to $RTI
    elsif ( ($RTI == 0) && ($DAISY == 0) ) {
	if (-d "/usr2/bbx") {
	    $RTI = 1;
	}
	elsif (-d "/d/daisy") {
	    $DAISY = 1;
	}
	else {
	    $RTI = 1;
	}
    }

    # one or the other was specified

    return(1);
}


sub tfr_tfsupport_account_name
{
    return($DEF_TFSUPPORT_ACCOUNT_NAME);
}

sub tfr_pos_admin_group_name
{
    return($DEF_RTI_ADMIN_GROUP_NAME) if ($RTI);
    return($DEF_DAISY_ADMIN_GROUP_NAME) if ($DAISY);
    return("root");
}

sub tfr_pos_group_name
{
    return($DEF_RTI_GROUP_NAME) if ($RTI);
    return($DEF_DAISY_GROUP_NAME) if ($DAISY);
    return("root");
}


############################
###                      ###
###    TEST FUNCTIONS    ###
###                      ###
############################

sub tfr_test_functions
{
    # --test-get-shopcode
    if ($TEST_GET_SHOPCODE) {
	exit(tfr_test_get_shopcode());
    }

    # --test-process-lock
    if ($TEST_PROCESS_LOCK) {
	exit(tfr_test_process_lock());
    }

    # --test-notify-no-backup-device
    if ($TEST_NOTIFICATION_NO_BACKUP_DEVICE) {
	exit(tfr_test_notification_no_backup_device());
    }

    # --test-gen-users-listing-file
    if ($TEST_GEN_USERS_LISTING_FILE) {
	exit(tfr_test_gen_users_listing_file());
    }

    # --test-gen-users-shadow-file
    if ($TEST_GEN_USERS_SHADOW_FILE) {
	exit(tfr_test_gen_users_shadow_file());
    }

    # --test-retry-backup
    if ($TEST_RETRY_BACKUP) {
	tfr_test_backup_retry();
    }

    # --test-gen-default-config-file
    if ($TEST_GEN_DEF_CONFIG_FILE) {
	exit(tfr_test_gen_default_config_file());
    }

    # --test-parse-rsync-stats
    if ($TEST_PARSE_RSYNC_STATS) {
	exit(tfr_test_parse_rsync_stats());
    }

    # --test-rsync-exit-status
    if ($TEST_RSYNC_EXIT_STATUS) {
	tfr_test_rsync_exit_status();
    }

    # --test-edit-auth-keys
    if ($TEST_EDIT_AUTH_KEYS) {
	exit(tfr_test_edit_auth_keys());
    }

    # --test-restore-upgrade-add-users
    if ($TEST_RESTORE_UPGRADE_ADD_USERS) {
	exit(tfr_test_restore_upgrade_add_users());
    }

    # --test-restore-upgrade-adjust-users
    if ($TEST_RESTORE_UPGRADE_ADJUST_USERS) {
	exit(tfr_test_restore_upgrade_adjust_users());
    }

    # --test-restore-upgrade-samba-conf
    if ($TEST_RESTORE_UPGRADE_SAMBA_CONF) {
	exit(tfr_test_restore_upgrade_samba_conf());
    }

    # --test-restore-upgrade-samba-passwd
    if ($TEST_RESTORE_UPGRADE_SAMBA_PASSWD) {
	exit(tfr_test_restore_upgrade_samba_passwd());
    }

    # --test-restore-upgrade-set-homedir-owners
    if ($TEST_RESTORE_UPGRADE_HOMEDIR_OWNERS) {
	exit(tfr_test_restore_upgrade_homedir_owners());
    }

    return(1);
}


sub tfr_test_get_shopcode
{
    my $shopcode = get_shopcode();
    if ($shopcode eq $EMPTY_STR) {
	print "could not get shopcode\n";
    }
    else {
	print "shopcode = $shopcode\n";
    }

    return($EXIT_OK);
}


sub tfr_test_process_lock
{
    $SERVER = 1;

    tfr_process_lock_obtain();

    my $pid = tfr_process_lock_pid();
    my $acquire_time = tfr_process_lock_acquire_time();

    print "process lock pid = $pid, process lock acquire time = $acquire_time\n";
    
    @EMAIL_RECIPIENTS = ("gsmith\@teleflora.com");
    $EMAIL_SERVER = "sendmail";
    $DeviceType = $DEVTYPE_SERVER;

    tfr_notification_process_lock_acquisition();

    tfr_process_lock_release();

    return($EXIT_OK);
}

sub tfr_test_notification_no_backup_device
{
    @EMAIL_RECIPIENTS = ("gsmith\@teleflora.com");
    $EMAIL_SERVER = "sendmail";

    tfr_notification_no_backup_device();

    return($EXIT_OK);
}

sub tfr_test_gen_users_listing_file
{
    my $list_users_cmd = tfrm_pathto_pos_users_script() . " --list";
    my $users_listing_file_path = tfrm_nameof_users_listing_file();

    tfr_generate_users_listing_file($list_users_cmd, $users_listing_file_path);

    return($EXIT_OK);
}

sub tfr_test_gen_users_shadow_file
{
    my $list_users_cmd = tfrm_pathto_pos_users_script() . " --list";
    my $users_listing_file_path = tfrm_nameof_users_listing_file();

    tfr_generate_users_listing_file($list_users_cmd, $users_listing_file_path);

    my $shadow_file_path = "/etc/shadow";
    my $users_shadow_file_path = tfrm_nameof_users_shadow_file();

    tfr_generate_users_shadow_file($users_listing_file_path, $shadow_file_path, $users_shadow_file_path);

    return($EXIT_OK);
}


sub tfr_test_backup_retry
{
    loginfo("***** testing backup retries *****");

    return(1);
}


sub tfr_test_gen_default_config_file
{
    my $config_file_path = tfrm_pathto_def_pos_config_file();
    my $config_file_name = basename($config_file_path);

    my $generated_config_file = tfr_install_default_config_file($config_file_name);
    if ($generated_config_file) {
	print "default config file generated: $generated_config_file\n";
    }
    else {
	print "could not generate default config file: $config_file_name\n";
    }

    return($EXIT_OK);
}


sub tfr_test_parse_rsync_stats
{
    tfr_record_rsync_stats($TEST_PARSE_RSYNC_LOG);

    print "rsync sent     = $RsyncStatsSent\n";
    print "rsync received = $RsyncStatsReceived\n";
    print "rsync rate     = $RsyncStatsRate\n";

    return($EXIT_OK);
}


sub tfr_test_rsync_exit_status
{
    loginfo("***** testing rsync exit status *****");

    return(1);
}


sub tfr_test_edit_auth_keys
{
    my $auth_keys_path = "authorized_keys";
    my $key_path = "id_rsa.pub";
    
    if (tfr_auth_keys_file_rm_key($auth_keys_path, $key_path)) {
	print "public key in $key_path removed from $auth_keys_path\n";
    }
    else {
	print "public key in $key_path NOT removed from $auth_keys_path\n";
    }

    return($EXIT_OK);
}


sub tfr_test_restore_upgrade_add_users
{
    my $users_listing_file = "users_listing.txt";
    tfr_restore_upgrade_add_users($users_listing_file);

    return($EXIT_OK);
}

sub tfr_test_restore_upgrade_adjust_users
{
    my $users_shadow_file = "users_shadow.txt";
    my $system_shadow_file = "shadow";
    tfr_restore_upgrade_adjust_users($users_shadow_file, $system_shadow_file);

    return($EXIT_OK);
}


sub tfr_test_restore_upgrade_samba_conf
{
    my $samba_conf_file = "smb.conf";
    tfr_restore_upgrade_samba_conf($samba_conf_file);

    return($EXIT_OK);
}


sub tfr_test_restore_upgrade_samba_passwd
{
    my $samba_pdb = 'smbpasswd';
    my $new_samba_pdb = $samba_pdb . '.' . $$;
    tfr_restore_upgrade_samba_gen_passwd($samba_pdb, $new_samba_pdb);

    return($EXIT_OK);
}


sub tfr_test_restore_upgrade_homedir_owners
{
    my $users_listing_file = "users_listing.txt";

    tfr_restore_upgrade_homedir_owners($users_listing_file);

    return($EXIT_OK);
}

###############################
###############################
###############################


sub shutdown_and_exit
{
    my ($exit_status) = @_;

    unless (tfr_log_conclude()) {
	print {*STDERR} "error closing log files\n";
    }

    exit($exit_status);
}


#
# small API for managing the rsync exit status.
#
# if code calling the backup transactions function
# gets a return value of $EXIT_RSYNC_ERROR,
# it can call the rsync status accessor to get the
# actual value returned by rsync.
#
sub tfr_rsync_exit_status_clear
{
    $RsyncExitStatus = 0;

    return(1);
}


sub tfr_rsync_exit_status_fetch
{
    return($RSYNC_EXIT_STATUS_IO_ERROR) if ($TEST_RETRY_BACKUP);

    return($RsyncExitStatus);
}


sub tfr_rsync_exit_status_record
{
    my ($exit_status) = @_;

    $RsyncExitStatus = $exit_status;

    return(1);
}


sub tfr_rsync_exit_status_prepare
{
    my ($returnval) = @_;

    if ($TEST_RETRY_BACKUP) {
	# return rsync error for all but last interation
	my $retry_backup_reps = tfr_retry_backup_reps();
	my $retries = tfr_backup_retry_fetch_retries();
	$returnval = ($retries < $retry_backup_reps) ? $EXIT_RSYNC_ERROR : $returnval;
    }

    return($returnval);
}


#
# exit status could be adjusted when testing is in effect -
# return $RSYNC_EXIT_STATUS_PARTIAL or
# $RSYNC_EXIT_STATUS_VANISHED every 5th call
#
# use a "static" variable to hold counter
BEGIN {
    my $counter = 0;
    sub tfr_rsync_exit_status_adjust
    {
	my ($rc) = @_;

	if ($TEST_RSYNC_EXIT_STATUS) {
	    if ($counter % 5 == 0) {
		$rc = (rand(100) > 50) ? $RSYNC_EXIT_STATUS_PARTIAL : $RSYNC_EXIT_STATUS_VANISHED;
	    }
	    $counter++;
	}

	return($rc);
    }
}

#
# if any rsync commands executed during backup operations
# returned a timeout error, value 30, then this accessor
# will return true.
#
sub tfr_rsync_status_timeout_seen
{
    return($RsyncTimeoutSeen);
}

sub tfr_rsync_status_timeout_record
{
    $RsyncTimeoutSeen = 1;

    return($RsyncTimeoutSeen);
}


#
# if any rsync commands executed during a backup operation
# returned an exit status that is not considered serious
# enough to abort will be considered a warning.
#
sub tfr_rsync_status_warnings_init
{
    @RsyncWarnings = ();

    return(1);
}

sub tfr_rsync_status_warnings_push
{
    my ($rsync_exit_status) = @_;

    unless (grep {/$rsync_exit_status/} @RsyncWarnings) {
	push(@RsyncWarnings, $rsync_exit_status);
    }

    return(1);
}

sub tfr_rsync_status_warnings_fetch
{
    return(@RsyncWarnings);
}


#
# classify value of rsync exit status and handle appropriately.
#
# Returns
#   rsync exit status
#
sub tfr_rsync_exit_status_classify
{
    my ($rc) = @_;

    # save the rsync status value
    tfr_rsync_exit_status_record($rc);

    # a value of 23 from rsync, which the man page
    # states is:  partial transfer due to error,
    # is only going to be a warning, so just note it and
    # reset status to success.
    if ($rc == $RSYNC_EXIT_STATUS_PARTIAL) {
	loginfo("warning: rsync command exit status: partial xfer due to error ($rc)");
	tfr_rsync_status_warnings_push($rc);
	$rc = 0;
    }
    
    # a value of 24 from rsync, which the rsync man page
    # states is: Partial transfer due to vanished source files,
    # is only going to be a warning, so just note it and
    # reset status to success.
    elsif ($rc == $RSYNC_EXIT_STATUS_VANISHED) {
	loginfo("warning: rsync command exit status: partial xfer due to vanished files ($rc)");
	tfr_rsync_status_warnings_push($rc);
	$rc = 0;
    }

    # a value of 255 from rsync is going to be
    # classified as an SSH connection error.
    elsif ($rc == $RSYNC_EXIT_STATUS_SSH_ERROR) {
	logerror("rsync command exit status: ssh connection error ($rc)");
	$rc = $EXIT_RSYNC_ERROR;
    }

    # a value of 30 from rsync, which the rsync man pages
    # states is: Timeout in data send/receive,
    # is going to be an error.
    elsif ($rc == $RSYNC_EXIT_STATUS_IO_ERROR) {
	tfr_rsync_status_timeout_record();
	logerror("rsync command exit status: timeout in data send/receive ($rc)");
	$rc = $EXIT_RSYNC_ERROR;
    }

    # anything other than 0 is an error
    elsif ($rc != 0) {
	showerror("rsync command exited with non-zero value: $rc");
	$rc = $EXIT_RSYNC_ERROR;
    }

    return($rc);
}


#
# small API for backup retries.
#
sub tfr_retry_backup_wait
{
    return($RetryBackupWait);
}

sub tfr_retry_backup_reps
{
    return($RetryBackupReps);
}

sub tfr_retry_backup_record_retries
{
    my ($backup_retry_iteration) = @_;

    $RetryBackupIterations = $backup_retry_iteration;

    return($RetryBackupIterations);
}

sub tfr_backup_retry_fetch_retries
{
    return($RetryBackupIterations);
}


#
# output elements of an array, with specified separator, and
# with specified number of elements per line.
#
sub tfr_print_array
{
    my ($array_ref, $leadchar, $sepchar, $elems_per_line) = @_;

    # if array empty, nothing to print
    unless (scalar(@{$array_ref})) {
	return(1);
    }

    # if elements per line <= 0, nothing to print
    if ($elems_per_line > 0) {
	my $elems_out_counter = 0;
	foreach my $elem (@{$array_ref}) {
	    if ($elems_out_counter >= $elems_per_line) {
		print "$sepchar\n";
		$elems_out_counter = 0;
	    }
	    print $leadchar if ($elems_out_counter == 0);
	    print $sepchar if ($elems_out_counter > 0);
	    print $elem;
	    $elems_out_counter++;
	}
	print "\n";
    }

    return(1);
}

#
# Convert seconds to human readable time.
#
# Taken from www.perlmonks.org
#
sub conv_time_to_dhms
{
    my ($s) = @_;

    return "less than 1 sec" if ($s == 0);

    return sprintf "00:00:%02d", $s if $s < 60;

    my $m = $s / 60; $s = $s % 60;
    return sprintf "00:%02d:%02d", $m, $s if $m < 60;

    my $h = $m /  60; $m %= 60;
    return sprintf "%02d:%02d:%02d", $h, $m, $s if $h < 24;

    my $d = $h / 24; $h %= 24;
    return sprintf "%d:%02d:%02d:%02d", $d, $h, $m, $s;
}

#
# get "yes/no" answer from stdin.
#
# Returns
#   1 if "y"
#   0 for anything else
#
sub get_yn
{
    my ($prompt) = @_;

    my $answer = "";

    while (1) {
	print("$prompt?\n");
	print("(Y/N) >");
	$answer = <>;
	if (defined($answer)) {
	    chomp($answer);
	    last if ($answer =~ /^n/i);
	    last if ($answer =~ /^y/i);
	}
	else {
	    # eof seen (usually ^D)
	    $answer = 'n';
	    last;
	}
    }

    if ($answer =~ /^y/i) {
	return(1);
    }

    return(0);
}


sub get_command_line
{
	my $cmd_line = "";

	$cmd_line = $0;
	foreach my $i (@ARGV) {
		$cmd_line .= " ";
		if ($i =~ /\s/) {
			if ($i =~ /(--[[:print:]]+)(=)(.+)$/) {
				$cmd_line .= "$1$2\"$3\"";
			}

			else {
				$cmd_line .= "\"$i\"";
			}
		}

		else {
			$cmd_line .= "$i";
		}
	}

	return($cmd_line);
}


sub is_configured_sendmail
{
    return($EMAIL_SERVER eq "sendmail");
}

sub is_configured_smtp_mail
{
    return( ($EMAIL_SERVER) && ($EMAIL_SERVER ne "sendmail") && $EMAIL_USER && $EMAIL_PASS );
}

sub is_configured_email_recipients
{
    return(scalar(@EMAIL_RECIPIENTS));
}

sub is_configured_print
{
    return(scalar(@PRINTERS));
}

#
# if the value of $RootDirCLO is equal to the default value,
# then a restore in place is being done.
#
sub is_restore_in_place
{
    return($RootDirCLO eq $DEF_ROOTDIR);
}

# PCI 6.5.6
#
# Look for a list of patterns in an input string that might
# indicate that some insecure value was passed into the script.
# Example, no input scring should have the BACKQUOTE chars as
# part of the string... if there were, it could mean that
# someone was trying to get the script to run another unknown
# script as "root".
#
# Returns
#   1 if input is insecure
#   0 if input is OK
#
sub is_input_insecure
{
    my ($cmd) = @_;

    return(0) if ($cmd eq "");

    my $rc = 0;

    my @insecure_patterns = (
	'\`',               # `bad command`
	'(\$\()(.*.)(\))',  # $(bad command)
	'\;',               # stuff ; bad command
	'\&',               # stuff && bad command
	'\|',               # stuff | bad command
	'\>',               # stuff > bad command
	'\<',               # stuff < bad command
	'[[:cntrl:]]',      # non printables
    );

    foreach my $re (@insecure_patterns) {
	if ($cmd =~ /$re/) {
	    $rc = 1;
	    last;
	}
    }

    return($rc);
}


sub is_service_configured
{
    my ($service_name) = @_;

    my $rc = 0;

    my $cmd = "/sbin/chkconfig --list";
    if (open(my $pipe, '-|', $cmd)) {
	while (<$pipe>) {
	    if (/^${service_name}\s+/) {
		$rc = 1;
	    }
	}
	close($pipe);
    }
    else {
	logerror("error opening pipe to command: $cmd");
    }

    return($rc);
}


#
# Get the file system UUID from the specified backup device.
#
# Returns
#   non-empty string with 36 character UUID
#   empty string if UUID can not be found
#
sub get_filesys_uuid
{
    my ($device) = @_;

    my $filesys_uuid  = OSTools::Filesys::filesys_uuid($device);

    return($filesys_uuid);
}


#
# Verify that the backup device path is either a block device or
# the path to an image file of at least a minimum size.
#
# Return TRUE if backup device path is verified
# Return FALSE if not
#
sub device_is_verified
{
    my ($device_path) = @_;

    if (-b $device_path) {
	return(1);
    }

    elsif (-f $device_path) {
	if ((-s $device_path) >= $DEVICE_IMAGE_FILE_MIN) {
	    return(1);
	}
	else {
	    logerror("Backup image file less then minimum size: $DEVICE_IMAGE_FILE_MIN bytes");
	}
    }

    else {
	logerror("Backup device must be a block device or an existing image file");
    }

    return(0);
}


#
# Format the backup device using Linux EXT2 Filesystem.
# Note that this supports either block devices, or image files.
#
# Returns
#   0 on success
#   non-zero on failure
#
sub tfr_device_format_ext2
{
    my ($device, $device_type) = @_;

    my $returnval = 0;
    my $answer = "";
    my $uuid = "";
    my $ml = "format";

    if ($device eq "") {
	logerror("[$ml] No device specified to format.");
	return(-2);
    }

    unless (device_is_verified($device)) {
	logerror("[$ml] Backup device path verification error");
	return(-3);
    }

    # Don't format a mounted drive.
    if (is_rsync_bu_device_mounted()) {
	logerror("[$ml] Backup Device is mounted. Please --unmount before formatting.");
	return(-3);
    }

    my $begin_timestamp = POSIX::strftime("%a %b %d %H:%M:%S %Y", localtime());

    showinfo("[$ml] Formatting $device_type: $device");


    # Determine the "old" UUID (if any).  If there is no old UUID,
    # then by setting the value to "random", it will cause the
    # tune2fs command to generate a new UUID for the file system.

    $uuid = get_filesys_uuid($DEVICE);
    if ($uuid eq "") {
	$uuid = "random";
    }

    #
    # Put a file system on the device.
    #
    # Note that, under RHEL5 at least, these same commands work equally
    # whether talking to a block device or a file. "mkfs.ext2" and "tune2fs"
    # transparently figure out whether loopback devices are needed.
    #
    # Make the disk label something that can be searched for and
    # recognized as a disk being used as a Teleflora backup device.
    # A disk label can be a maximum of 16 chars, so make it:
    #
    #       |<---- 16 ---->|
    #       |     chars    |
    #	    TFBUDSK-YYYYMMDD
    #
    # which would be the string "TFBUDSK" followed by a date stamp.
    # "TFBUDSK" stands for "Teleflora Backup Disk".  The disk label
    # can be read with the e2label command.

    my $label_brand = "TFBUDSK";
    my $label_datestamp = POSIX::strftime("%Y%m%d", localtime());
    my $label_disk = $label_brand . "-" . $label_datestamp;

    showinfo("[$ml] Making an EXT2 file system on $device_type: $device");
    my $fmt_cmd = "/sbin/mkfs.ext2 -F -L $label_disk $device";
    if ($VERBOSE) {
	# all output to logs and stdout
	showinfo("[$ml] $fmt_cmd");
	system("$fmt_cmd 2>&1 | tee -a $LOGFILE");
    }
    else {
	# all output to logs
	loginfo("[$ml] $fmt_cmd");
	system("$fmt_cmd >> $LOGFILE 2>> $LOGFILE");
    }
    $returnval = $?;
    if ($returnval != 0) {
	showerror("[$ml] Could not format: $device");
    }
    else {
	showinfo("[$ml] Setting tuning parameters for $device_type: $device");
	my $tune_cmd = "/sbin/tune2fs -c 0 -e remount-ro -i 12m -U $uuid $device";
	if ($VERBOSE) {
	    showinfo("[$ml] Setting max mount count to 0, thus disabling it");
	    showinfo("[$ml] Setting error behavior to \"remount read-only on error\"");
	    showinfo("[$ml] Setting FSCK interval to 12 months");
	    showinfo("[$ml] Setting UUID of file system to: $uuid");
	    system("$tune_cmd 2>&1 | tee -a $LOGFILE");
	    $returnval = $?;
	}
	else {
	    loginfo("[$ml] $tune_cmd");
	    system("$tune_cmd >> $LOGFILE 2>> $LOGFILE");
	    $returnval = $?;
	}
	if ($returnval != 0) {
	    showerror("[$ml] Could not tune2fs: $device");
	}

	else {

	    my $end_timestamp = POSIX::strftime("%a %b %d %H:%M:%S %Y", localtime());

	    showinfo("[$ml] Setting up $PROGNAME framework on $device_type: $device");

	    #
	    # Write format info to newly formatted device
	    #
	    if (mount_device("rw")) {
		logerror("[$ml] Could not mount $device_type after formatting: $device");
		$returnval = -4;
	    }
	    else {

		showinfo("[$ml] Writing tag file to $device_type: $device");

		my $format_info = << "END_FORMAT_INFO";
#
#      Backup program: $PROGNAME $CVS_REVISION
#         Device type: $device_type $device
#      Format Started: $begin_timestamp
#    Format Completed: $end_timestamp
#
END_FORMAT_INFO

		if (open(my $BUSIG, ">", "$MOUNTPOINT/$FORMAT_FILE")) {
		    print($BUSIG "$format_info");
		    close($BUSIG);
		}
		else {
		    logerror("Could not write backup tag file: $MOUNTPOINT/$FORMAT_FILE");
		    $returnval = -5;
		}

		unmount_device();
	    }
	}
    }

    return($returnval);
}


#
# remove a list of paths.
#
# if path does not exist, do nothing and don't complain.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_runtime_cleanup_unlink_paths
{
    my (@paths) = @_;

    my $rc = 1;

    for my $path (@paths) {
	if (-e $path) {
	    unlink $path;
	    if (-e $path) {
		showerror("runtime cleanup could not remove: $path");
		$rc = 0;
	    }
	    else {
		loginfo("runtime cleanup removed: $path");
	    }
	}
    }

    return($rc);
}


sub tfr_runtime_cleanup
{
    my $rc = $EXIT_OK;

    my $process_lock_path = tfr_pathto_process_lock();
    my @paths = ($process_lock_path);

    if ($SERVER || $CLOUD) {
	my $ssh_socket_path = tfr_pathto_ssh_tunnel_socket();
	push(@paths, $ssh_socket_path);
    }

    if (tfr_runtime_cleanup_unlink_paths(@paths)) {
	loginfo("runtime cleanup successful");
    }
    else {
	logerror("could not cleanup runtime files");
	$rc = $EXIT_RUNTIME_CLEANUP;
    }

    return($rc);
}


##################################
###                            ###
###   PROCESS LOCK FUNCTIONS   ###
###                            ###
##################################

#
# called from mainline - try to obtain a process lock.
# if process lock not available, cleanup and exit.
#
# if "--dry-run" specified, just return.
#
sub tfr_process_lock_obtain
{
    return(1) if ($DRY_RUN);

    if (tfr_process_lock_setup()) {
	my $process_lockfile_path = tfr_process_lock_acquire();
	if ($process_lockfile_path) {
	    loginfo("process lock acquired: $process_lockfile_path");
	}
	else {
	    showerror("could not acquire process lock: can not continue");
	    tfr_notification_process_lock_acquisition();
	    shutdown_and_exit($EXIT_LOCK_ACQUISITION);
	}
    }
    else {
	showerror("could not set up process lock: can not continue");
	shutdown_and_exit($EXIT_LOCK_SETUP);
    }

    return(1);
}


sub tfr_process_lock_classify
{
    my $lockfile_type = "";
    if ($SERVER) {
	$lockfile_type = $LOCKFILE_TYPE_SERVER;
    }
    elsif ($CLOUD) {
	$lockfile_type = $LOCKFILE_TYPE_CLOUD;
    }
    elsif ($DEVICE) {
	$lockfile_type = $LOCKFILE_TYPE_DEVICE;
    }
    else {
	$lockfile_type = $LOCKFILE_TYPE_UNK;
    }

    return($lockfile_type);
}


#
# If the process lock file directory does not exist,
# and is the public lock directory, log an error message and
# return an error.  If a private lock file directory,
# try to make it if it does not exist, and log an error
# message if the mkdir fails.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_process_lock_mkdir
{
    my $rc = 1;

    # public lockfile directory
    if ($LOCKFILE_DIR eq "/var/lock") {
	unless (-d $LOCKFILE_DIR) {
	    logerror("directory for process lock file does not exist: $LOCKFILE_DIR");
	    $rc = 0;
	}
    }

    # private lockfile directory
    else {
	unless (-d $LOCKFILE_DIR) {
	    system("mkdir $LOCKFILE_DIR");
	    unless (-d $LOCKFILE_DIR) {
		logerror("could not make process lock directory: $LOCKFILE_DIR");
		$rc = 0;
	    }
	}
    }

    return($rc);
}

#
# return the path to the process lock lockfile.
#
# The lockfile name is formed from the lockfile dir and
# device type.
#
# Returns
#   path to lockfile on success
#   empty string on error
#
sub tfr_pathto_process_lock
{
    my $lockfile_type = tfr_process_lock_classify();
    my $lockfile_path = $LOCKFILE_DIR . '/' . $DEF_PROJ_NAME;
    $lockfile_path .= '-' . $lockfile_type;
    $lockfile_path .= $LOCKFILE_SUFFIX;

    return($lockfile_path);
}

#
# init the process lock.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_process_lock_setup
{
    my $rc = 1;

    unless (tfr_process_lock_mkdir()) {
	logerror("lockfile dir does not exist and can't make it: $LOCKFILE_DIR");
	$rc = 0;
    }

    return($rc);
}

#
# attempt to get exclusive access to the process lock.
#
# Returns
#   path to lockfile if successful
#   empty string if not
#
# FIXME: if making the lock file fails, look for a pid file and
# if there is a process that corresponds to the pid.  If there
# is no pid file, look for a "edir_update.pl" process by name.
# if there is no currently running instance, there should not be
# a lock file or a pid file.
#
sub tfr_process_lock_acquire
{
    my $rc = $EMPTY_STR;

    if ($SERVER || $CLOUD || $DEVICE) {
	my $lockfile_path = tfr_pathto_process_lock();
	if ($DEBUGMODE) {
	    loginfo("attempting to acquire process lock: $lockfile_path");
	}
	if (sysopen(LFH, $lockfile_path, O_EXCL|O_CREAT)) {
	    close(LFH);
	    if (open(my $lfh, '>', $lockfile_path)) {
		print($lfh "$$\n");
		close($lfh);
		if ($DEBUGMODE) {
		    loginfo("process lock acquired: $lockfile_path");
		    loginfo("pid written to process lock file: $$");
		}
	    }
	    else {
		logerror("could not write pid to process lock file: $lockfile_path");
	    }

	    $rc = $lockfile_path;
	}

	# couldn't get lock, but verify lock is valid.
	# get pid from lockfile and verify the process
	# still exists.
	else {
	    if (open(my $lfh, '<', $lockfile_path)) {
		my $pid = <$lfh>;
		chomp($pid);
		close($lfh);
		unless (kill(0, $pid)) {
		    showerror("could not verify validity of existing process lock: $lockfile_path");
		    showerror("pid contained in process lock does not exist: $pid");
		}
	    }
	    else {
		showerror("could not verify lockfile validity: $lockfile_path");
	    }
	}
    }

    return($rc);
}

#
# release the process lock.
#
# Returns
#   1 if successful
#   0 if not
#
sub tfr_process_lock_release
{
    my $rc = 1;

    return($rc) if ($DRY_RUN);

    if ($SERVER || $CLOUD || $DEVICE) {
	my $lockfile_path = tfr_pathto_process_lock();

	if (-f $lockfile_path) {
	    unless (unlink($lockfile_path)) {
		logerror("error unlinking process lock $lockfile_path: $!");
		$rc = 0;
	    }
	}
	else {
	    loginfo("process lock already released, file does not exist: $lockfile_path");
	}
    }

    return($rc);
}

sub tfr_process_lock_pid
{
    my $pid = 0;

    my $lockfile_path = tfr_pathto_process_lock();
    if (-e $lockfile_path) {
	if (open(my $lfh, '<', $lockfile_path)) {
	    $pid = <$lfh>;
	    chomp($pid);
	    close($lfh);
	}
	else {
	    loginfo("could not get process lock pid, could not open file: $lockfile_path");
	}
    }
    else {
	loginfo("could not get process lock pid, file does not exist: $lockfile_path");
    }

    return($pid);
}

sub tfr_process_lock_acquire_time
{
    my $acquire_time = "";

    my $lockfile_path = tfr_pathto_process_lock();
    if (-e $lockfile_path) {
	my $st = File::stat::stat($lockfile_path);
	my $lockfile_mtime = POSIX::ctime($st->mtime);
	chomp($lockfile_mtime);
	$acquire_time = $lockfile_mtime;
    }
    else {
	loginfo("could not get process lock mtime, file does not exist: $lockfile_path");
    }

    return($acquire_time);
}


####################################
###                              ###
###        NOTIFICATIONS         ###
###                              ###
####################################

sub tfr_notification_process_lock_acquisition
{
    my $timestamp = POSIX::strftime("%Y-%m-%d", localtime());
    my $subject = "$PROGNAME could not start new $DeviceType instance on $timestamp";

    my $pid = tfr_process_lock_pid();
    my $acquire_time = tfr_process_lock_acquire_time();

    my $message = "$subject.\n";
    $message .= "Could not acquire the process lock for new instance of $PROGNAME.\n";
    if ($pid != 0) {
	$message .= "Previous instance with PID $pid started at $acquire_time still running.\n";
    }
    $message .= "\n";
    $message .= "$PROGNAME\n";

    return(tfr_send_email($subject, $message));
}


sub tfr_notification_no_backup_device
{
    my $timestamp = POSIX::strftime("%Y-%m-%d", localtime());
    my $subject = "$PROGNAME could not perform a backup on $timestamp - No Backup Device";
    my $message = "\n";
    $message .= "-- Backup Device Not Found --\n";
    $message .= "\n";
    $message .= "A backup could not be performed because a device could not be found to\n";
    $message .= "store the backup.\n";
    $message .= "\n";
    $message .= "If you are using a Passport drive, verify that the drive is connected to\n";
    $message .= "the server by a  USB cable, and verify that the small white light is\n";
    $message .= "illuminated on the side of the Passport drive.\n";
    $message .= "\n";
    $message .= "$PROGNAME\n";

    tfr_send_email($subject, $message);

    return(tfr_print_results($subject, $message));
}


sub tfr_notification_backup_error
{
    my ($backup_rc) = @_;

    my $subject = "Error notification from backup function of $PROGNAME";
    my $message = "The $PROGNAME backup function returned error: $backup_rc\n";
    $message .= "Description of error: $ExitTable{$backup_rc}\n";
    if ($backup_rc == $EXIT_RSYNC_ERROR) {
	my $rsync_exit_status = tfr_rsync_exit_status_fetch();
	$message .= "The rsync command returned: $rsync_exit_status\n";
    }

    return(tfr_send_email($subject, $message));
}


sub get_free_space
{
    my ($mount_point) = @_;

    my $available = 0;
    my $ref = OSTools::Filesys::filesys_df($mount_point);

    if (exists($ref->{available})) {
	$available = $ref->{available};
    }
    else {
	showerror("error getting available space of filesystem at: $mount_point");
    }

    return($available);
}


#
# Public IP Address
#
sub get_public_ip_addr
{
    my $publicip = "";
    my $cmd = "curl --silent http://icanhazip.com";

    # retry several times
    for (my $i=1; $i <= 5; $i++) {
	if (open(my $pipe, '-|', $cmd)) {
	    while (<$pipe>) {
		chomp;
		if (/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/) {
		    $publicip = $1;
		}
	    }
	    close($pipe);
	    if ($publicip) {
		loginfo("iterations to obtain public ip: $i");
		last;
	    }
	}
	else {
	    logerror("error opening pipe to: $cmd");
	}
    }

    return($publicip);
}


#
# get the ip address of the default or CLI specified network interface.
#
# Returns
#   non-empty string on success
#   empty string on error
#
sub get_ip_address
{
    my $ip_addr = get_network_attribute($NetworkDeviceCLO, "inet addr");

    return($ip_addr);
}


#
# get the netmask of the default or CLI specified network interface.
#
# Returns
#   non-empty string on success
#   empty string on error
#
sub get_netmask
{
    my $netmask = get_network_attribute($NetworkDeviceCLO, "Mask");

    return($netmask);
}


#
# get a network attribute - one of ip addr, broadcast address, or
# netmask.
#
# Returns
#   non-empty string on success
#   empty string on error
#
sub get_network_attribute
{
    my ($device, $pattern) = @_;

    my $rc = "";

    my $cmd = "/sbin/ifconfig $device 2> /dev/null";
    if (open(my $pipe, '-|', $cmd)) {
	while (<$pipe>) {
	    if (/${pattern}:(\d+\.\d+\.\d+\.\d+)/) {
		$rc = $1;
	    }
	}
	close($pipe);
    }
    else {
	logerror("error opening pipe to command: $cmd");
    }

    return($rc);
}


#
# get the gateway address
#
# Returns
#   non-empty string on success
#   empty string on error
#
sub get_gateway_ipaddr
{
    my $rc = "";

    my $route_cmd = "/sbin/route -n";
    my $pattern = "0.0.0.0";
    my @route_table_entry = ();

    if (open(my $pipe, '-|', $route_cmd)) {
	while (<$pipe>) {
	    next until(/^$pattern/);
	    @route_table_entry = split(/\s+/);
	    last;
	}
	close($pipe);
    }
    else {
	logerror("error opening pipe to command: $route_cmd");
    }

    # check for a route table entry with something reasonable in it
    if (exists($route_table_entry[1])) {
	my $gateway = $route_table_entry[1];
	if ($gateway =~ /\d+\.\d+\.\d+\.\d+/) {
	    $rc = $gateway;
	}
	else {
	    logerror("unrecognized format for gateway address: $rc");
	}
    }
    else {
	logerror("unexpected output of route command: $route_cmd");
    }

    return($rc); 
}


#
# verify network device - all this means is that "ifconfig"
# returned an exit status of 0 for this interface.
#
# Returns
#   1 if verify successful
#   0 if not
#
sub verify_network_device
{
    my ($network_device) = @_;

    my $rc = 1;

    system("/sbin/ifconfig $network_device > /dev/null 2>&1");
    if ($? != 0) {
	$rc = 0;
    }

    return($rc);
}


#
# search file for regular expression
#
# Returns
#   0 found regular expression
#   1 did not find regular expression
#
sub fgrep
{
    my ($fpath, $re) = @_;

    my $rc = 1;
    if (open(my $fp, '<', $fpath)) {
	while (<$fp>) {
	    chomp;
	    if (/$re/) {
		$rc = 0;
		last;
	    }
        }
	close($fp);
    }

    return($rc);
}


sub system_service_disable
{
    my ($service_name) = @_;

    system("/sbin/chkconfig --list | grep $service_name");
    if ($? == 0) {
	system("/sbin/service $service_name stop");
	system("/sbin/chkconfig $service_name off");
    }

    return(1);
}


#
# Get info about the backup set on the backup device
#
sub get_backup_device_info
{
    my ($device, $mountpoint) = @_;

    loginfo("Get backup device info");

    # Verify the backup device can be mounted.
    mount_device("ro");
    unless (is_rsync_bu_device_mounted()) {
	if (-f $device) {
	    showinfo("Could not mount image file: $device");
	    showinfo("Use \"--format\" to format image file first");
	}
	else {
	    showinfo("Could not mount backup device: $device");
	}
	return(1);
    }

    my $uuid = get_filesys_uuid($device);

    print("Backup device file system UUID: $uuid\n");

    my $free_space = get_free_space($mountpoint);
    print("Backup device free space: $free_space KB\n");

    unmount_device();

    loginfo("---- END Get Info about Backups ----");

    return(0);
}


#
# Daisy POS can have one or more daisy db dirs.
#
sub get_daisy_db_dirs
{
    my @daisy_db_dirs = ();

    # if not a Daisy system, then list is empty, we are done
    unless (-d "/d/daisy") {
	return(@daisy_db_dirs);
    }

    my @d_dirs = glob("/d/*");

    for my $d_dir (@d_dirs) {

	# must be a directory
	next unless (-d $d_dir);

	# skip old daisy dirs
	next if ($d_dir =~ /.+-\d{12}$/);

	# must contain the magic files
	next unless(-e "$d_dir/flordat.tel");
	next unless(-e "$d_dir/control.dsy");

	# must be daisy 8.0+
	next unless (-d "$d_dir/bin");

	push(@daisy_db_dirs, $d_dir);
    }

    return(@daisy_db_dirs);
}


#
# get the shopcode
#
# Returns:
#   shopcode as 8 digit string on success
#   empty string on error
#
sub get_shopcode
{
    my $shopcode = "";

    my $rti_dir = tfr_pathto_rti_dir();
    my $daisy_dir = tfr_pathto_daisy_dir();

    if (-d $rti_dir) {
	my $dove_cfg_file = tfr_pathto_rti_shopcode_file();
	if (-f $dove_cfg_file) {
	    if (open(my $df, "<", $dove_cfg_file)) {
		while (<$df>) {
		    if (/DOVE_USERNAME\s*=\s*([[:print:]]+)/) {
			$shopcode = $1;
			last;
		    }
		}
		close($df);
	    }
	}
	else {
	    logerror("RTI shopcode file does not exist: $dove_cfg_file");
	}
    }
    elsif (-d $daisy_dir) {
	my $dove_cfg_file = tfr_pathto_daisy_shopcode_file();
	if (-f $dove_cfg_file) {
	    if (open(my $df, "<", $dove_cfg_file)) {
		my $buffer;
		my $rc = sysread($df, $buffer, 38);
		if (defined($rc) && $rc != 0) {
		    $shopcode = substr($buffer, 30, 8);
		}
		close($df);
	    }
	}
	else {
	    logerror("Daisy shopcode file does not exist: $dove_cfg_file");
	}
    }
    else {
	logerror("could not get shopcode: POS not installed");
    }

    return($shopcode);
}


############################################
##                                        ##
## subsection: "pathto" and "nameof" subs ##
##                                        ##
############################################

sub tfr_pathto_rti_dir
{
    return($RTIDIR);
}

sub tfr_pathto_rti_shopcode_file
{
    return($RTI_SHOPCODE_FILE);
}

sub tfr_pathto_daisy_dir
{
    return($DAISYDIR);
}

sub tfr_pathto_daisy_shopcode_file
{
    return($DAISY_SHOPCODE_FILE);
}

sub tfrm_pathto_harden_linux_cmd
{
    return("$RTI_TOOLS_BINDIR/harden_linux.pl") if ($RTI);
    return("$DAISY_TOOLS_BINDIR/harden_linux.pl") if ($DAISY);
    return("/teleflora/ostools/bin/harden_linux.pl");
}

sub tfrm_pathto_project_bu_dir
{
    return($DEF_RTI_RSYNC_BU_DIR) if ($RTI);
    return($DEF_DAISY_RSYNC_BU_DIR) if ($DAISY);
    return($EMPTY_STR);
}

sub tfrm_pathto_def_pos_config_file
{
    return($POS_CONFIG_FILE_PATH);
}

sub tfrm_pathto_def_pos_config_dir
{
    return($POS_CONFIG_DIR_PATH);
}

sub tfrm_pathto_config_file
{
    return($CONFIGFILE);
}

sub tfrm_pathto_logfile_dir
{
    return($LogfileDir);
}

sub tfrm_pathto_logfile
{
    return($LOGFILE);
}

sub tfrm_pathto_summary_logfile
{
    return($SummaryLogfile);
}

sub tfrm_pathto_debug_logfile
{
    return($DebugLogfile);
}

sub tfrm_pathto_users_info_dir
{
    my $bu_dir = tfrm_pathto_project_bu_dir();
    return($bu_dir . '/' . $USERS_INFO_DIR) if ($bu_dir);
    return($EMPTY_STR);
}

sub tfrm_pathto_perm_file
{
    my ($bu_type) = @_;

    my $perm_file_dir = tfrm_pathto_project_bu_dir();
    my $perm_file_name = tfrm_nameof_perm_file($bu_type);
    my $bu_perm_file_path = $perm_file_dir . '/' . $perm_file_name;

    return($bu_perm_file_path);
}

sub tfrm_pathto_pos_users_script
{
    return("$RTI_BINDIR/rtiuser.pl") if ($RTI);
    return("$DAISY_BINDIR/dsyuser.pl") if ($DAISY);
    return($EMPTY_STR);
}

sub tfrm_pathto_pserver_info_dir
{
    return(tfrm_pathto_project_bu_dir() . '/' . $PSERVER_INFO_DIR);
}

sub tfrm_pathto_pserver_info_file
{
    return(tfrm_pathto_pserver_info_dir() . '/' . $PSERVER_INFO_FILE);
}

sub tfrm_pathto_restored_pserver_info_dir
{
    return(File::Spec->catdir(tfrm_pathto_pserver_info_dir(),
			      tfrm_nameof_restored_pserver_info_dir()));
}

sub tfrm_pathto_restored_pserver_info_file
{
    return(File::Spec->catdir(tfrm_pathto_restored_pserver_info_dir(),
			      tfrm_nameof_pserver_info_file()));
}

sub tfrm_pathto_buserver_info_dir
{
    return(tfrm_pathto_project_bu_dir() . '/' . $BUSERVER_INFO_DIR);
}

sub tfrm_pathto_bu_server_info_file
{
    return(tfrm_pathto_buserver_info_dir() . '/' . $BUSERVER_INFO_FILE);
}

sub tfrm_pathto_pserver_cloister_dir
{
    return(tfrm_pathto_project_bu_dir() . '/' . $PSERVER_CLOISTER_DIR);
}

#
# return the path to production server transfer dir.
#
# Returns
#   path to xfer dir on success
#   partial path if error
#
sub tfrm_pathto_pserver_xfer_dir
{
    my $tfsupport = tfr_tfsupport_account_name();
    my $homedir = tfr_accounts_homedir($tfsupport);
    return($homedir . '/' . $PSERVER_XFER_DIR);
}

sub tfrm_pathto_users_listing_file
{
    my $pathto_users_info_dir = tfrm_pathto_users_info_dir();
    my $nameof_users_listing_file = tfrm_nameof_users_listing_file();
    my $pathto_users_listing_file = File::Spec->catdir($pathto_users_info_dir, $nameof_users_listing_file);
    return($pathto_users_listing_file);
}

sub tfrm_pathto_users_shadow_file
{
    my $pathto_users_info_dir = tfrm_pathto_users_info_dir();
    my $nameof_users_shadow_file = tfrm_nameof_users_shadow_file();
    my $pathto_users_shadow_file = File::Spec->catdir($pathto_users_info_dir, $nameof_users_shadow_file);
    return($pathto_users_shadow_file);
}

sub tfr_pathto_ssh_tunnel_socket
{
    my $ssh_socket_path = $EMPTY_STR;

    if ( ($DeviceType eq $DEVTYPE_SERVER) || ($DeviceType eq $DEVTYPE_CLOUD) ) {
	my $ssh_socket_dir_path = $DEF_SSH_SOCKET_DIR_PATH;
	my $ssh_socket_file_name = $DEF_SSH_SOCKET_PREFIX . $DeviceType . $DEF_SSH_SOCKET_EXT;
	$ssh_socket_path = File::Spec->catdir($ssh_socket_dir_path, $ssh_socket_file_name);
    }

    return($ssh_socket_path);
}


#
# Given an account name, return the path to the ssh id file.
#
# Returns
#   path to file on success
#   empty string on error
#
sub tfr_pathto_ssh_id_file
{
    my ($account_name) = @_;

    my $ssh_id_path = $EMPTY_STR;

    my $sshdir_path = tfr_sshdir_default_path($account_name);
    if ($sshdir_path) {
	$ssh_id_path = File::Spec->catdir($sshdir_path, $SSH_KEY_FILENAME);
    }

    return($ssh_id_path);
}


#
# Given an account name, form the path to the authorized keys file.
#
# Returns
#   path to authorized keys file success
#   empty string on error
#
sub tfr_pathto_ssh_auth_keys_file
{
    my ($account_name) = @_;

    my $auth_keys_path = $EMPTY_STR;

    my $sshdir_path = tfr_sshdir_default_path($account_name);
    if ($sshdir_path) {
	$auth_keys_path = File::Spec->catdir($sshdir_path, $SSH_AUTH_KEYS_FILENAME);
    }

    return($auth_keys_path);
}


#
# Given an account name, return the path to the public key file.
#
# Input
#   account name
#
# Returns
#   path to key file on success
#   empty string on error
#
sub tfr_pathto_ssh_public_key_file
{
    my ($account_name) = @_;

    my $public_key_path = $EMPTY_STR;

    my $sshdir_path = tfr_sshdir_default_path($account_name);
    if ($sshdir_path) {
	$public_key_path = File::Spec->catdir($sshdir_path, $SSH_KEY_FILENAME_PUBLIC);
    }

    return($public_key_path);
}


#
# given a backup type, return the name of the perm file
# for that type.
#
# always returns a non-empty string.
#
sub tfrm_nameof_perm_file
{
    my ($bu_type) = @_;

    my $perm_file_name = $bu_type . tfrm_suffixof_perm_file();

    return($perm_file_name);
}

#
# return name of users listing file.
#
sub tfrm_nameof_users_listing_file
{
    return($RTI_USERS_LISTING_FILE) if ($RTI);
    return($DAISY_USERS_LISTING_FILE) if ($DAISY);
    return($EMPTY_STR);
}

#
# return name of users shadow file.
#
sub tfrm_nameof_users_shadow_file
{
    return($RTI_USERS_SHADOW_FILE) if ($RTI);
    return($DAISY_USERS_SHADOW_FILE) if ($DAISY);
    return($EMPTY_STR);
}

sub tfrm_nameof_pserver_info_file
{
    return($PSERVER_INFO_FILE);
}

sub tfrm_nameof_restored_pserver_info_dir
{
    return($RESTORED_PSERVER_INFO_DIR);
}

sub tfrm_suffixof_perm_file
{
    return($PERM_FILE_SUFFIX);
}


#
# produce a perm file for the specified files of the specified
# backup type.  Put it in the tfrsync backup dir.
#
# Save the results in a file whose name is the backup type
# with the perm file suffix.  Put the file in directory
# "/usr2/tfrsync" for RTI systems, "/d/tfrsync" for Daisy
# systems.
#
# To get the metadata, run the command "getfacl --absolute-names -R"
# on each backup item and concatenate text to a perms
# file.
#
# Returns
#   path to perms file on success
#   empty string on error
#
sub tfr_produce_perm_file
{
    my ($tobackup, $bu_type) = @_;

    my $bu_perm_file_path = tfrm_pathto_perm_file($bu_type);
    if (scalar(@{$tobackup})) {
	print "\n";
	showinfo("saving output to permfile: $bu_perm_file_path");
    }

    foreach my $src (@{$tobackup}) {
	if ($DRY_RUN) {
	    system("echo \"getfacl --absolute-names -R $src >> $bu_perm_file_path\"");
	}
	else {
	    my $cmd = "getfacl --absolute-names -R $src";
	    showinfo("command: $cmd");
	    system("$cmd >> $bu_perm_file_path");
	    my $exit_status = $?;
	    if ($exit_status == -1) {
		showerror("command failed to execute: $!");
	    }
	    elsif ($exit_status & 127) {
		my $signo = ($exit_status & 127);
		showerror("command died from signal: $signo");
	    }
	    else {
		$exit_status = ($exit_status >> 8);
		if ($exit_status != 0) {
		    showerror("command exit status non-zero: $exit_status");
		}
	    }
	}
    }

    return($bu_perm_file_path);
}


#
# top level sub for "--generate-permfiles"
#
# generate the perm files for a set of backup types.
#
# Returns
#   $EXIT_OK on success
#   $EXIT_GENERATE_PERMS if error generating perm file
#
sub tfr_generate_perm_files
{
    my (@bu_type_list) = @_;

    my $rc = $EXIT_OK;

    foreach my $bu_type (@bu_type_list) {
	my @toback = tfr_backup_file_list($bu_type);
	my $perm_file = tfr_produce_perm_file(\@toback, $bu_type);
	unless ($perm_file) {
	    showerror("error generating perm file for backup type: $bu_type");
	    $rc = $EXIT_GENERATE_PERMS;
	    last;
	}
    }

    return($rc);
}


#
# top level sub for "--upload-permfiles"
#
# upload any perm files located in the tfrsync backup dir
# to the cloud server.
#
# Returns
#   $EXIT_OK on success
#   $EXIT_SSH_ID_FILE if path to ssh id file does not exist
#   $EXIT_SSH_TUNNEL_OPEN if error opening master socket
#   $EXIT_UPLOAD_PERMS if error uploading a perm file
#   $EXIT_SSH_TUNNEL_CLOSE if error closing master socket
#
sub tfr_upload_perm_files
{
    my ($account_name, $cloud_server) = @_;

    my $rc = $EXIT_OK;

    my $rsync_bu_dir = tfrm_pathto_project_bu_dir();
    my $bu_perm_files_re = $rsync_bu_dir . "/*" . tfrm_suffixof_perm_file();

    my @bu_perm_file_path_list = glob($bu_perm_files_re);
    if ($DRY_RUN) {
	foreach (@BACKUP_TYPES) {
	    my $bu_perm_file_path = $rsync_bu_dir . '/' . $_ . tfrm_suffixof_perm_file();
	    push(@bu_perm_file_path_list, $bu_perm_file_path);
	}
    }

    if (scalar(@bu_perm_file_path_list) == 0) {
	showinfo("there are no perm files to upload: $bu_perm_files_re");
	return($EXIT_OK);
    }

    # open the ssh tunnel socket
    unless (tfr_open_ssh_tunnel($account_name, $cloud_server)) {
	showerror("could not open ssh tunnel socket for perm upload: $account_name, $cloud_server");
	return($EXIT_SSH_TUNNEL_OPEN);
    }

    # form the rsync command
    my $rsync_cmd = tfr_construct_rsync_cmd($account_name, $DEVTYPE_CLOUD);

    # prefix for rsync source argument
    my $dst_prefix = $account_name . "@" . $cloud_server . ":";

    foreach my $bu_perm_file_path (@bu_perm_file_path_list) {
	my $src = $bu_perm_file_path;
	my $dst = $dst_prefix . basename($bu_perm_file_path);

	if ($DRY_RUN) {
	    system("echo \"$rsync_cmd $src $dst\"");
	}
	else {
	    system("$rsync_cmd $src $dst 2>> $LOGFILE");
	    if ($? != 0) {
		showerror("could not upload perm file: $bu_perm_file_path");
		$rc = $EXIT_UPLOAD_PERMS;
		last;
	    }
	}
    }

    unless (tfr_close_ssh_tunnel($account_name, $cloud_server)) {
	showerror("could not close ssh tunnel socket for perm upload: $account_name, $cloud_server");
	if ($rc == $EXIT_OK) {
	    $rc = $EXIT_SSH_TUNNEL_CLOSE;
	}
    }

    return($rc);
}


sub tfr_construct_rsync_cmd
{
    my ($account_name, $devtype) = @_;

    # start forming the rsync command
    my $rsync_cmd = "rsync -ahv";

    # add ssh identity file and ssh tunnel socket
    my $ssh_id_path = tfr_pathto_ssh_id_file($account_name);
    my $ssh_tunnel_socket_path = tfr_pathto_ssh_tunnel_socket();
    $rsync_cmd .= " -e \'ssh -i $ssh_id_path -o ControlPath=$ssh_tunnel_socket_path\'";

    # dry run or not
    if ($RSYNC_TRIAL) {
	$rsync_cmd .= " -n";
    }

    # add any extra options specified in config file
    foreach (@RSYNC_OPTIONS) {
	$rsync_cmd .= " $_";
    }

    return($rsync_cmd);
}


#
# get a perm file at the specifed source path on the
# cloud server using the specified rsync command and
# put it at the specified destination path.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_get_perm_file
{
    my ($rsync_cmd) = @_;

    my $rc = 0;

    # exec the remote command
    if ($DRY_RUN) {
	system("echo \"$rsync_cmd\"");
    }
    else {
	loginfo("Execing rsync command:");
	loginfo($rsync_cmd);

	system("$rsync_cmd 2>> $LOGFILE");
	if ($? == 0) {
	    $rc = 1;
	}
	else {
	    showerror("error returned by rsync command: $?");
	}
    }

    return($rc);
}


#
# top level sub for "--download-permfiles"
#
# For the specified account on the specified cloud server,
# get the perm file for each of the specified backup types.
#
# Returns
#   $EXIT_OK on success
#   $EXIT_SSH_ID_FILE if ssh identity file not found 
#   $EXIT_SSH_TUNNEL_OPEN if open of ssh tunnel fails
#   $EXIT_DOWNLOAD_PERMS if download fails
#
sub tfr_download_perm_files
{
    my ($account_name, $cloud_server, @bu_type_list) = @_;

    my $rc = $EXIT_OK;

    # open the ssh tunnel socket
    unless (tfr_open_ssh_tunnel($account_name, $cloud_server)) {
	showerror("could not open ssh tunnel socket for perm download: $account_name, $cloud_server");
	return($EXIT_SSH_TUNNEL_OPEN);
    }

    # form the rsync command
    my $rsync_cmd = tfr_construct_rsync_cmd($account_name, $DEVTYPE_CLOUD);

    # prefix for rsync source argument
    my $src_prefix = $account_name . "@" . $cloud_server . ":";

    # for each backup type, download corresponding perm file
    foreach my $bu_type (@bu_type_list) {
	my $src_path = $src_prefix . tfrm_nameof_perm_file($bu_type);
	my $dst_path = tfrm_pathto_perm_file($bu_type);
	
	$rsync_cmd .= " $src_path $dst_path";

	unless (tfr_get_perm_file($rsync_cmd)) {
	    showerror("could not download perm file: $dst_path");
	    $rc = $EXIT_DOWNLOAD_PERMS;
	    last;
	}
    }

    if (tfr_close_ssh_tunnel($account_name, $cloud_server)) {
	loginfo("ssh tunnel socket for perm download closed: $account_name, $cloud_server");
    }
    else {
	showerror("could not close ssh tunnel socket for perm download: $account_name, $cloud_server");
	if ($rc == $EXIT_OK) {
	    $rc = $EXIT_SSH_TUNNEL_CLOSE;
	}
    }

    return($rc);
}


#
# top level sub for "--restore-from-permfiles"
#
# for the specified account on the cloud server, set
# the perms for the specified backup type.
#
# Returns
#   $EXIT_OK on success
#   $EXIT_RESTORE_PERMS if perms could not be restored
#   $EXIT_PERM_FILE_MISSING if perm file does not exist
#
sub tfr_restore_from_perm_files
{
    my ($account_name, $cloud_server, @bu_type_list) = @_;

    my $rc = $EXIT_OK;

    foreach my $bu_type (@bu_type_list) {
	my $perm_file_path = tfrm_pathto_perm_file($bu_type);
	if ($perm_file_path) { 
	    print "restoring perms from $perm_file_path...";
	    if (tfr_restore_perms_from_perm_file($perm_file_path)) {
		print "\n";
		loginfo("perms restored from: $perm_file_path");
	    }
	    else {
		logerror("error restoring perms from perm file: $perm_file_path");
		$rc = $EXIT_RESTORE_PERMS;
	    }
	}
	else {
	    logerror("perm file for backup type does not exist: $bu_type");
	    $rc = $EXIT_PERM_FILE_MISSING;
	}

	last if ($rc != $EXIT_OK);
    }

    return($rc);
}


#
# restore perms from specified perm file.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_restore_perms_from_perm_file
{
    my ($perm_file_path) = @_;

    my $rc = 1;

    if ($DRY_RUN) {
	system("echo \"setfacl --restore=$perm_file_path\"");
    }
    else {
	system("setfacl --restore=$perm_file_path 2> /dev/null");
	if ($? != 0) {
	    logerror("non-zero error status returned by setfacl: $?");
	    $rc = 0;
	}
    }

    return($rc);
}


sub tfr_list_longest_elem
{
    my (@list_elems) = @_;

    my $longest_elem = $list_elems[0];
    my $elem_len = length $longest_elem;
    for my $elem (@list_elems) {
	if (length($elem) > $elem_len ) {
	    $longest_elem = $elem;
	    $elem_len = length($elem);
	}
    }

    return($longest_elem);
}


#
# Add excludes for all the daisy database dirs to the global
# list of excluded paths.
#
sub determine_daisy_excludes
{
    my ($global_excludes) = @_;

    my @daisy_db_dirs = get_daisy_db_dirs();

    foreach my $daisy_db_dir (@daisy_db_dirs) {
	push(@{$global_excludes}, "${daisy_db_dir}-*");
	push(@{$global_excludes}, "${daisy_db_dir}/*.iso");
	push(@{$global_excludes}, "${daisy_db_dir}/*.tar.asc");
	push(@{$global_excludes}, "${daisy_db_dir}/backups/*");
	#push(@{$global_excludes}, "${daisy_db_dir}/log/*");
    }

    return(1);
}


sub backup_dev_capacity
{
    mount_device("ro");
    unless (is_rsync_bu_device_mounted()) {
	logerror("Could not mount backup device: $DEVICE");
	return(-1);
    }

    my $ref = OSTools::Filesys::filesys_df($MOUNTPOINT);
    my $blocks = 0;

    if (exists($ref->{blocks})) {
	$blocks = $ref->{blocks};
    }

    unmount_device();

    return($blocks);
}


sub backup_dev_available
{
    mount_device("ro");
    unless (is_rsync_bu_device_mounted()) {
	logerror("Could not mount backup device: $DEVICE");
	return(-1);
    }

    my $ref = OSTools::Filesys::filesys_df($MOUNTPOINT);
    my $available = 0;

    if (exists($ref->{available})) {
	$available = $ref->{available};
    }

    unmount_device();

    return($available);
}


#
# process backup summary report
#
# 1. generate a report
# 2. save it
# 3. send it via email if configured
#
sub tfr_backup_summary_report
{
    my ($begin_time, $end_time, $backup_rc) = @_;

    my $summary = tfr_backup_summary_report_generate($begin_time, $end_time, $backup_rc);
    if ($summary) {
	if (tfr_backup_summary_report_save($summary)) {
	    my $subject = ($backup_rc == $EXIT_OK) ? "SUCCESSFUL" : "ERROR";
	    $subject .= " summary report from $PROGNAME";
	    if ($SEND_SUMMARY) {
		tfr_send_email($subject, $summary);
	    }
	    if (is_configured_print()) {
		if (tfr_print_results($subject, $summary)) {
		    loginfo("summary report printed");
		}
		else {
		    logerror("could not print summary report");
		}
	    }
	}
	else {
	    logerror("could not save backup summary report");
	}
    }
    else {
	logerror("could not generate formatted backup summary report");
    }

    return(1);
}


#
# generate a backup summary report entry
#
# Each entry of the summary log file contains:
#
#   name of script
#   version of script
#   command line
#   time of start of execution
#   time of end of execution
#   duration of execution
#   device type
#   result description
#   rsync exit status
#   rsync backup retries
#   rsync warnings
#   rsync bytes sent
#   server ip addr, either backup server or cloud, else NA
#   path on server, either backup server or device, else NA
#   device file if device type is "passport", "rev", "usb", or "image"
#   capacity if device type is "passport", "rev", "usb", or "image"
#   available if device type is "passport", "rev", "usb", or "image"
#
# Entry separator is line of 80 EQUAL chars ('=')
#
sub tfr_backup_summary_report_generate
{
    my ($begin_time, $end_time, $backup_rc) = @_;

    my $rec_sep = "=" x 80;
    my $summary = "";

    my $begin_timestamp = POSIX::strftime("%Y%m%d-%H%M%S", localtime($begin_time));
    my $end_timestamp = POSIX::strftime("%Y%m%d-%H%M%S", localtime($end_time));
    my $duration = $end_time - $begin_time;
    my $duration_timestamp = conv_time_to_dhms($duration);

    my $backup_rc_desc = $ExitTable{$backup_rc};

    my $rsync_exit_status = tfr_rsync_exit_status_fetch();
    my $backup_retries = tfr_backup_retry_fetch_retries();

    $summary .= "$rec_sep\n";
    $summary .= "    PROGRAM: $PROGNAME\n";
    $summary .= "    VERSION: $CVS_REVISION\n";
    $summary .= "    COMMAND: $COMMAND_LINE\n";
    $summary .= "      BEGIN: $begin_timestamp\n";
    $summary .= "        END: $end_timestamp\n";
    $summary .= "   DURATION: $duration_timestamp\n";
    $summary .= "     DEVICE: $DeviceType\n";
    $summary .= "     RESULT: $backup_rc_desc\n";
    $summary .= "      RSYNC: $rsync_exit_status\n";
    $summary .= "    RETRIES: $backup_retries\n";

    my @rsync_warnings = tfr_rsync_status_warnings_fetch();
    $summary .= "   WARNINGS: ";
    $summary .= (@rsync_warnings) ? "@rsync_warnings" : "0";
    $summary .= "\n";

    my $rsync_bytes_sent = tfr_format_rsync_stats($RsyncStatsSent);
    $summary .= " BYTES SENT: $rsync_bytes_sent\n";

    $summary .= "     SERVER: ";
    if ($DeviceType eq $DEVTYPE_SERVER) {
	$summary .= $RsyncServer;
    }
    elsif ($DeviceType eq $DEVTYPE_CLOUD) {
	$summary .= $CLOUD_SERVER;
    }
    else {
	$summary .= "NA";
    }
    $summary .= "\n";

    $summary .= "       PATH: ";
    if ( ($DeviceType eq $DEVTYPE_SERVER) ||
         ($DeviceType eq $DEVTYPE_FILE_SYSTEM) ) {
	$summary .= ($RsyncDir eq $EMPTY_STR) ? "(in place)" : $RsyncDir;
    }
    else {
	$summary .= "NA";
    }
    $summary .= "\n";

    if ($DeviceType eq $DEVTYPE_PASSPORT ||
	$DeviceType eq $DEVTYPE_REV ||
	$DeviceType eq $DEVTYPE_USB ||
	$DeviceType eq $DEVTYPE_IMAGE) {
	$summary .= "DEVICE FILE:$DEVICE\n";
	my $capacity = backup_dev_capacity();
	$summary .= "   CAPACITY: $capacity\n";
	my $available = backup_dev_available();
	$summary .= "  AVAILABLE: $available\n";
    }
    else {
	$summary .= "DEVICE FILE: NA\n";
	$summary .= "   CAPACITY: NA\n";
	$summary .= "  AVAILABLE: NA\n";
    }

    $summary .= "$rec_sep\n";
    $summary .= "\n";

    return($summary);
}

#
# save the current summary report to the summary log file
#
# Returns
#   1 on success
#   0 on errror
#
sub tfr_backup_summary_report_save
{
    my ($summary) = @_;

    my $rc = 1;

    return($rc) if ($DRY_RUN);

    if (open(my $slf, ">>", $SummaryLogfile)) {
	if (log_file_lock($slf)) {
	    print {$slf} $summary;
	    log_file_unlock($slf);
	}
	else {
	    logerror("error obtaining file lock on: $SummaryLogfile");
	    $rc = 0;
	}
	close($slf);
    }
    else {
	logerror("Could not open summary log file: $SummaryLogfile");
	$rc = 0;
    }

    return($rc);
}


#
# get the list of bu types appropriate to the current platform.
#
# Returns
#   non-empty list on success
#   empty list on error
#
sub tfr_get_platform_bu_types
{
    my (@bu_type_list) = @_;

    my @platform_bu_types = ();

    foreach my $bu_type (@bu_type_list) {
	next if ($bu_type eq $BU_TYPE_ALL);
	next if ($RTI && $bu_type eq $BU_TYPE_DAISY);
	next if ($RTI && $bu_type eq $BU_TYPE_DAISY_CONFIGS);
	next if ($DAISY && $bu_type eq $BU_TYPE_USR2);
	next if ($DAISY && $bu_type eq $BU_TYPE_RTI_CONFIGS);
	push(@platform_bu_types, $bu_type);
    }

    return(@platform_bu_types);
}


sub tfr_log_rsync_transaction
{
    my ($tobackup, $excludes, $bu_type, $transaction_class) = @_;

    # Collect some information about this backup set.	
    loginfo("|=======================");
    loginfo("| rsync transaction info");
    loginfo("|=======================");
    loginfo("| rsync Started: " . localtime(time()));
    loginfo("| rsync PID: $$");

    # What is our hostname?
    my $hostname = hostname();
    loginfo("| Hostname: $hostname");

    # Which version of redhat does this file come from?
    loginfo("| Red Hat Version: " . plat_redhat_version());

    # 32 or 64 bit linux?
    loginfo("| Architecture: " . plat_processor_arch());

    # Which Linux Kernel?
    loginfo("| Kernel: " . plat_kernel_release());

    # What is the backup type?
    loginfo("| Backup Type: $bu_type");

    # What is the transaction class?
    loginfo("| Transaction Class: $transaction_class");

    # What are we backing up?
    loginfo("| ToBackup: @{$tobackup}");

    # What are we excluding?
    loginfo("| Exclude: @{$excludes}");

    loginfo("|=======================");

    return(1);
}


#
# Fill a file with paths to exclude from rsync transaction.
# If the exclude prefix is not empty and is not present on
# the front of the exclude path, add to the front of each path.
#
# Returns
#   name of temp file if there are files to exclude
#   empty string if there are NO files to exclude
#
sub tfr_rsync_exclude_file
{
    my ($excludes, $exclude_prefix) = @_;
    my $returnval = "";

    if (scalar(@{$excludes})) {
	my $template = "tfrsync-excludes-XXXXXXX";
	my ($tfh, $tfn) = tempfile($template, DIR => '/tmp');
	loginfo("tempfile for rsync excludes: $tfn");

	foreach (@{$excludes}) {
	    if ($exclude_prefix) {
		unless (/^$exclude_prefix/) {
		    $_ = $exclude_prefix . $_;
		}
	    }
	    print {$tfh} "$_\n";
	}
	close($tfh);
	$returnval = $tfn;
    }

    return($returnval);
}


sub tfr_set_sigint_handler
{
    local $SIG{'INT'} = 'tfr_sigint_handler';

    $SigIntSeen = 0;

    return(1);
}


sub tfr_reset_sigint_handler
{
    local $SIG{'INT'} = 'DEFAULT';

    return(1);
}


sub tfr_sigint_handler
{
    $SigIntSeen = 1;

    return(1);
}


#
# open an ssh tunnel
#
# if this function succeeds, then the "pathto" function for
# the ssh tunnel socket returns a path to the active socket.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_open_ssh_tunnel
{
    my ($account_name, $server) = @_;

    return(1) if ($DRY_RUN);

    my $rc = 0;

    my $ssh_id_path = tfr_pathto_ssh_id_file($account_name);
    unless (-e $ssh_id_path) {
	logerror("[open ssh tunnel] could not get path to ssh identity file: $account_name");
	return($rc);
    }

    # if ssh tunnel socket exists, attempt to remove it
    my $ssh_tunnel_socket_path = tfr_pathto_ssh_tunnel_socket();
    if (-e $ssh_tunnel_socket_path) {
	loginfo("[open ssh tunnel] ssh tunnel socket exists: $ssh_tunnel_socket_path");
	system("rm $ssh_tunnel_socket_path");
	if ($? == 0) {
	    loginfo("[open ssh tunnel] rm ssh tunnel socket ok: $ssh_tunnel_socket_path");
	}
	else {
	    logerror("[open ssh tunnel] ssh tunnel socket rm failed: $ssh_tunnel_socket_path");
	    return($rc);
	}
    }

    my $ssh_cmd = 'ssh -tfnN';
    $ssh_cmd .= " -i $ssh_id_path";
    $ssh_cmd .= " -o ControlMaster=yes";
    $ssh_cmd .= " -o ControlPath=$ssh_tunnel_socket_path";
    $ssh_cmd .= ' ' . $account_name . '@' . $server;

    loginfo("[open ssh tunnel] ssh command: $ssh_cmd");

    for (my $i=0; $i < $DEF_SSH_TUNNEL_RETRIES; $i++) {
	system("$ssh_cmd >> $LOGFILE 2>> $LOGFILE");

	my $ssh_exit_status = $?;
	if ($ssh_exit_status == -1) {
	    showerror("[open ssh tunnel] ssh command failed to execute: $!");
	    last;
	}
	elsif ($ssh_exit_status & 127) {
	    my $signo = ($ssh_exit_status & 127);
	    showerror("[open ssh tunnel] ssh command died from signal: $signo");
	    last;
	}
	else {
	    $ssh_exit_status = ($ssh_exit_status >> 8);

	    if ($ssh_exit_status == 0) {
		loginfo("[open ssh tunnel] ssh tunnel socket path: $ssh_tunnel_socket_path");
		$rc = 1;
		last;
	    }
	    elsif ($ssh_exit_status == 255) {
		# per man page for ssh
		showerror("[open ssh tunnel] could not open ssh tunnel: $ssh_tunnel_socket_path");
		showerror("[open ssh tunnel] open ssh tunnel command exit status: $ssh_exit_status"); 
		# try again
	    }
	    else {
		showerror("[open ssh tunnel] remote side exit status ssh tunnel command: $ssh_exit_status");
		last;
	    }
	}
	my $iteration = $i + 1;
	loginfo("[open ssh tunnel] ssh open tunnel socket iteration: $iteration");
    }

    return($rc);
}


#
# if there has been a rsync timeout error, where rsync has
# an exit status of 30, emperical evidence shows that the
# ssh tunnel socket is removed... in that case, log it as
# info rather than an error.
#
sub tfr_close_ssh_tunnel
{
    my ($account_name, $server) = @_;

    return(1) if ($DRY_RUN);

    my $rc = 0;

    my $ssh_id_path = tfr_pathto_ssh_id_file($account_name);
    unless (-e $ssh_id_path) {
	logerror("close ssh tunnel could not get path to ssh identity file: $account_name");
	return($rc);
    }

    my $ssh_tunnel_socket_path = tfr_pathto_ssh_tunnel_socket();
    unless (-e $ssh_tunnel_socket_path) {
	# if there was a rsync timeout, not an error for the
	# socket to not exist.
	if (tfr_rsync_status_timeout_seen()) {
	    loginfo("ssh tunnel socket does not exist after rsync timeout: $ssh_tunnel_socket_path");
	    return(1);
	}
	else {
	    logerror("ssh tunnel socket does not exist: $ssh_tunnel_socket_path");
	    return(0);
	}
    }

    my $ssh_cmd = 'ssh -t';
    $ssh_cmd .= " -O exit";
    $ssh_cmd .= " -i $ssh_id_path";
    $ssh_cmd .= " -o ControlPath=$ssh_tunnel_socket_path";
    $ssh_cmd .= ' ' . $account_name . '@' . $server;

    loginfo("close ssh tunnel command: $ssh_cmd"); 

    system("$ssh_cmd >> $LOGFILE 2>> $LOGFILE");

    my $ssh_exit_status = $?;
    if ($ssh_exit_status == -1) {
	showerror("close ssh tunnel command failed to execute: $!");
    }
    elsif ($ssh_exit_status & 127) {
	my $signo = ($ssh_exit_status & 127);
	showerror("close ssh tunnel command died from signal: $signo");
    }
    else {
	$ssh_exit_status = ($ssh_exit_status >> 8);
	if ($ssh_exit_status == 0) {
	    # as long as close was successful, return true
	    loginfo("close ssh tunnel succeeded: $ssh_tunnel_socket_path");
	    $rc = 1;
	    # if socket still exists, attempt to remove it but it is
	    # not an error if it can't be removed.
	    if (-e $ssh_tunnel_socket_path) {
		system("rm $ssh_tunnel_socket_path");
		if ($? == 0) {
		    loginfo("[close ssh tunnel] rm ssh socket ok: $ssh_tunnel_socket_path");
		}
		else {
		    loginfo("[close ssh tunnel] oops: ssh socket still exists: $ssh_tunnel_socket_path");
		}
	    }
	}
	else {
	    showerror("could not close ssh tunnel: $ssh_tunnel_socket_path");
	    showerror("close ssh tunnel command exit status: $ssh_exit_status"); 
	}
    }

    return($rc);
}


#
# read a users listing file and record the list of users
# in the given hash; also note in the hash whether the
# user is a normal user or an admin user.
#
# the content of a users listing file is assumed to be in
# the format as produced by the output of the "--list" option
# of either the "rtiuser.pl" or "dsyuser.pl" scripts.
#
# Returns
#   1 for success
#   0 for error
#
sub tfr_parse_users_listing_file
{
    my ($users_table, $users_listing_file_path) = @_;

    my $rc = 1;

    if (open(my $uf_fh, '<', $users_listing_file_path)) {
	while (my $line = <$uf_fh>) {
	    if ($line =~ /^(\S+)\s/) {
		$users_table->{$1} = 1;
	    }
	    if ($line =~ /^(\S+)\s.*RTI Admin/) {
		$users_table->{$1} = 2;
	    }
	    if ($line =~ /^(\S+)\s.*Daisy Admin/) {
		$users_table->{$1} = 2;
	    }
	}
	close($uf_fh);
    }
    else {
	logerror("could not open users listing file: $users_listing_file_path");
	$rc = 0;
    }

    return($rc);
}


#
# Generate a list of POS users and save it to a location
# that will be backed up.  This info is required for 
# transforming a backup server into a primary server.
#
# Returns
#   1 for success
#   0 for error
#
sub tfr_generate_users_listing_file
{
    my ($list_users_cmd, $users_listing_file_path) = @_;

    my $rc = 1;

    system("$list_users_cmd > $users_listing_file_path 2>> $LOGFILE");
    my $exit_status = $?;
    if ($exit_status == 0) {
	loginfo("generated new users listing file: $users_listing_file_path");
    }
    else {
	$exit_status = ($exit_status >> 8);
	showerror("could not generate users listing file, $list_users_cmd returned: $exit_status");
	$rc = 0;
    }

    return($rc);
}


#
# Generate a file of shadow file entries that correspond
# to the entries in the users listing file and save it to
# a location that will be backed up.  This info is required
# for transforming a backup server into a primary server.
#
# Returns
#   1 for success
#   0 for error
#
sub tfr_generate_users_shadow_file
{
    my ($users_listing_file_path, $shadow_file_path, $users_shadow_file_path) = @_;

    my $rc = 1;

    my %users_table = ();
    if (tfr_parse_users_listing_file(\%users_table, $users_listing_file_path)) {
	loginfo("successfully parsed users listing file: $users_listing_file_path");
	if (open(my $src_fh, '<', $shadow_file_path)) {
	    if (open(my $dst_fh, '>', $users_shadow_file_path)) {
		my $counter = 0;
		while (my $line = <$src_fh>) {
		    my ($username) = split(/:/, $line);
		    if (defined($users_table{$username})) {
			print {$dst_fh} $line;
			$counter++;
		    }
		}
		close($dst_fh);
		loginfo("generated new users shadow file: $users_shadow_file_path");
		loginfo("number of entries written to new users shadow file: $counter");
	    }
	    else {
		logerror("could not open-for-write users shadow file: $users_shadow_file_path");
		$rc = 0;
	    }
	    close($src_fh);
	}
	else {
	    logerror("could not open-for-read shadow file: $shadow_file_path");
	    $rc = 0;
	}
    }
    else {
	showerror("could not parse users listing file: $users_listing_file_path");
	$rc = 0;
    }

    unless (-s $users_shadow_file_path) {
	showerror("could not generate users shadow file: $users_shadow_file_path");
	$rc = 0;
    }

    return($rc);
}


#
# save users info:
#   1) generate and save the users listing file
#   2) generate and save the users shadow file
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_save_users_info
{
    my $rc = 1;

    my $users_info_dir_path = tfrm_pathto_users_info_dir();
    if (-d $users_info_dir_path) {
	my $users_listing_file_path = tfrm_pathto_users_listing_file();
	my $list_users_cmd = tfrm_pathto_pos_users_script() . " --list";
	if (tfr_generate_users_listing_file($list_users_cmd, $users_listing_file_path)) {
	    loginfo("users listing file saved: $users_listing_file_path");
	}
	else {
	    logerror("could not save users listing file: $users_listing_file_path");
	    $rc = 0;
	}
	my $users_shadow_file_path = tfrm_pathto_users_shadow_file();
	my $shadow_file_path = "/etc/shadow";
	if (tfr_generate_users_shadow_file( $users_listing_file_path,
					    $shadow_file_path,
					    $users_shadow_file_path)) {
	    loginfo("users shadow file saved: $users_shadow_file_path");
	}
	else {
	    logerror("could not save users shadow file: $users_shadow_file_path");
	    $rc = 0;
	}
    }
    else {
	logerror("users info directory does not exist: $users_info_dir_path");
	$rc = 0;
    }

    return($rc);
}


#
# generate a new pserver info file which will be
# copied to the backup server if a "--backup=all"
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_save_pserver_info
{
    my $rc = 1;

    my $pserver_infodir_path = tfrm_pathto_pserver_info_dir();
    if (-d $pserver_infodir_path) {
	my $pserver_info_file_path = tfrm_pathto_pserver_info_file();
	if (tfr_generate_pserver_info_file($pserver_info_file_path)) {
	    loginfo("new pserver info file saved: $pserver_info_file_path");
	}
	else {
	    logerror("could not save pserver info file: $pserver_info_file_path");
	    $rc = 0;
	}
    }
    else {
	logerror("pserver info directory does not exist: $pserver_infodir_path");
	$rc = 0;
    }

    return($rc);
}


#
# copy the pserver cloister files to the pserver
# cloister dir
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_save_pserver_cloister_files
{
    my $rc = 1;

    my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
    if (-d $pserver_cloister_dir_path) {
	my @cloistered_files = (
	    "/var/spool/cron",
	    "/etc/cron.d",
	);

	# extra file for RTI system
	if (-e $RTI_DOVE_CMD) {
	    push(@cloistered_files, $RTI_DOVE_CMD);
	}

	# update the files
	foreach (@cloistered_files) {
	    #system("tar cf - $_ | (cd $pserver_cloister_dir_path && tar xf -)");
	    system("cd /; rsync -ahv --delete --relative $_  $pserver_cloister_dir_path");
	    if ($? == 0) {
		loginfo("pserver cloister file copied to $pserver_cloister_dir_path: $_");
	    }
	    else {
		logerror("could not copy pserver cloister file to $pserver_cloister_dir_path: $_");
		$rc = 0;
	    }
	}
    }

    return($rc);
}


#
# add the default paths to *not* backup to the global
# backup exclude list.
#
sub tfr_backup_init_default_excludes
{
    push(@EXCLUDES, @DEF_BACKUP_EXCLUDES);

    if ($RTI) {
	push(@EXCLUDES, @DEF_RTI_BACKUP_EXCLUDES);
    }

    if ($DAISY) {
	push(@EXCLUDES, @DEF_DAISY_BACKUP_EXCLUDES);

	# exclude some files from within daisy database dirs
	determine_daisy_excludes(\@EXCLUDES);
    }

    return(1);
}


#
# do a backup of the POS log directory so the tfrsync.pl log files
# that were just written get copied to the backup device.  For the
# case of cloud backup, this will make it easier to collect the
# results by management facilities like Kaseya.
#
sub tfr_backup_pos_log_files
{
    my $rc = 1;

    my @types_to_backup = ( $BU_TYPE_POS_LOG_FILES );
    my $backup_rc = tfr_backup_files(\@types_to_backup, \@EXCLUDES);
    if ($backup_rc == $EXIT_OK) {
	loginfo("log files resynced via backup type: $BU_TYPE_POS_LOG_FILES");
    }
    else {
	logerror("could not rersync log files via backup type: $BU_TYPE_POS_LOG_FILES");
	$rc = 0;
    }

    return($rc);
}


#
# do a backup of the POS summary log file so content just written
# by tfrsync.pl gets copied to the backup device.  For the case
# of cloud backup, this will make it easier to collect the results
# by management facilities like Kaseya.
#
sub tfr_backup_pos_summary_log
{
    my $rc = 1;

    my @types_to_backup = ( $BU_TYPE_POS_SUMMARY_LOG );
    my $backup_rc = tfr_backup_files(\@types_to_backup, \@EXCLUDES);
    if ($backup_rc == $EXIT_OK) {
	loginfo("summary log file resynced via backup type: $BU_TYPE_POS_SUMMARY_LOG");
    }
    else {
	logerror("could not rersync summary log file via backup type: $BU_TYPE_POS_SUMMARY_LOG");
	$rc = 0;
    }

    return($rc);
}


############################################
##                                        ##
## subsection: restore upgrade            ##
##                                        ##
############################################


sub tfr_restore_upgrade_samba_check_conf
{
    my ($samba_conf_file) = @_;

    my $rc = 0;

    my $parameter = "passdb backend = smbpasswd";
    my $parameter2 = "smb passwd file = /etc/samba/smbpasswd";

    if (fgrep($samba_conf_file, $parameter) == 0) {
	if (fgrep($samba_conf_file, $parameter2) == 0) {
	    showinfo("[restore upgrade] samba config file already modified: $samba_conf_file");
	    $rc = 1;
	}
    }

    return($rc);
}


sub tfr_restore_upgrade_samba_gen_conf
{
    my ($samba_conf_file) = @_;

    my $rc = 1;

    my $parameter = "passdb backend = smbpasswd";
    my $parameter2 = "smb passwd file = /etc/samba/smbpasswd";

    my $new_samba_conf_file = $samba_conf_file . '.' . $$;

    #
    # Copy all lines from old to new, but immediately after the
    # global section declaraion, write the new parameter(s) into
    # the new conf file.
    #
    if (open(my $old_fh, '<', $samba_conf_file)) {
	if (open(my $new_fh, '>', $new_samba_conf_file)) {
	    while (<$old_fh>) {
		if (/^\s*\[global\]/) {
		    print {$new_fh} $_;
		    print {$new_fh} "#Following 2 lines added by $PROGNAME, $CVS_REVISION, $TIMESTAMP\n";
		    print {$new_fh} "$parameter\n";
		    print {$new_fh} "$parameter2\n";
		}
		else {
		    print {$new_fh} $_;
		}
	    }
	    close($new_fh);
	    close($old_fh);
	}
	else {
	    showerror("[restore upgrade] could not open new samba config file: $new_samba_conf_file");
	    $rc = 0;
	}
    }
    else {
	showerror("[restore upgrade] could not open existing samba config file: $samba_conf_file");
	$rc = 0;
    }

    if ($rc == 1) {
	# If the new conf file exists and is size non-zero, call it good
	# so replace the old one with the new.
	if (-e $new_samba_conf_file && -s $new_samba_conf_file) {
	    system("chmod --reference=$samba_conf_file $new_samba_conf_file");
	    system("chown --reference=$samba_conf_file $new_samba_conf_file");
	    rename $new_samba_conf_file, $samba_conf_file;
	    loginfo("[restore upgrade] samba config file updated successfully: $samba_conf_file");
	}
	else {
	    showerror("[restore upgrade] could not modify existing samba config file: $samba_conf_file");
	    unlink($new_samba_conf_file) if (-e $new_samba_conf_file);
	    $rc = 0;
	}
    }

    return($rc);
}


#
# verify that the Samba config file has been updated for RHEL6.
# If it has not, update it.  This must be done for RHEL6 systems
# to be backwards compatabile with the way the pre-RHEL6 systems
# were configured.
#
# Updating it consists of setting two parameters in the global
# section:
#   1) set the "passdb backend" parameter to a value of "smbpasswd"
#   2) set the path to the "smb passwd file" parameter
#
sub tfr_restore_upgrade_samba_conf
{
    my ($samba_conf_file) = @_;

    my $rc = 1;

    if (-e $samba_conf_file) {
	if (-f $samba_conf_file) {
	    if (tfr_restore_upgrade_samba_check_conf($samba_conf_file)) {
		loginfo("[restore upgrade] samba config file already modified: $samba_conf_file");
	    }
	    else {
		if (tfr_restore_upgrade_samba_gen_conf($samba_conf_file)) {
		    loginfo("[restore upgrade] samba config file updated: $samba_conf_file"); 
		}
		else {
		    showerror("[restore upgrade] could not update samba config file: $samba_conf_file");
		    $rc = 0;
		}
	    }
	}
	else {
	    showerror("[restore upgrade] samba config file not a regular file: $samba_conf_file");
	    $rc = 0;
	}
    }
    else {
	showerror("[restore upgrade] samba config file does not exist: $samba_conf_file");
	$rc = 0;
    }

    return($rc);
}


#
# Copy all lines from old to new, but adjust the UID field if necessary.
#
# The structure of an smbpasswd entry is:
#
# username:uid:lanman_hash:nt_hash:flags:pw_lct
#
# username    - the user's login name.
# uid         - the user's UID
# lanman_hash - Windows stuff
# nt_hash     - Windows stuff
# flags       - Various single-character flags representing the type and
#               state of the user's account.
# pw_lct      - the timestamp of the user's last successful password change
#
sub tfr_restore_upgrade_samba_gen_passwd
{
    my ($samba_pdb, $new_samba_pdb) = @_;

    my $rc = 1;

    if (open(my $old_fh, '<', $samba_pdb)) {
	if (open(my $new_fh, '>', $new_samba_pdb)) {
	    while (<$old_fh>) {
		my $line = $_;
		if ($line =~ /^(\S+):(\d+):(.*)$/) {
		    my $username = $1;
		    my $uid = $2;
		    my $remainder = $3;

		    my $system_uid = getpwnam($username);
		    if (defined($system_uid)) {
			if ($uid ne $system_uid) {
			    $line = "$username" . ":" . "$system_uid" . ":" . "$remainder" . "\n";
			}
		    }
		}
		print {$new_fh} $line;
	    }
	    close($new_fh);
	}
	else {
	    showerror("[restore upgrade] could not open new samba passwd file: $new_samba_pdb");
	    $rc = 0;
	}
	close($old_fh);
    }
    else {
	showerror("[restore upgrade] could not open existing samba passwd file: $samba_pdb");
	$rc = 0;
    }

    return($rc);
}


#
# Make the UIDs in the "smbpasswd" file match those in /etc/passwd.
#
sub tfr_restore_upgrade_samba_rebuild_passwd
{
    my ($samba_pdb) = @_;

    my $rc = 1;

    if (-f $samba_pdb) {

	my $new_samba_pdb = $samba_pdb . '.' . $$;

	if (tfr_restore_upgrade_samba_gen_passwd($samba_pdb, $new_samba_pdb)) {

	    # check result
	    if (-z $new_samba_pdb) {

		# bad: new generated conf file is zero sized
		showerror("[restore upgrade] generated samba passdb is a zero size: $new_samba_pdb");
		showerror("[restore upgrade] samba passwd file NOT modified: $samba_pdb");
		unlink $new_samba_pdb;
		$rc = 0;
	    }
	    else {

		# good: new conf file successfully generated, so replace old with new
		system("chmod --reference=$samba_pdb $new_samba_pdb");
		system("chown --reference=$samba_pdb $new_samba_pdb");
		rename $new_samba_pdb, $samba_pdb;

		loginfo("[restore upgrade] samba passwd file modified with new UID fields: $samba_pdb ");
	    }
	}
    }
    else {
	showerror("[restore upgrade] samba passswd file does not exist: $samba_pdb");
	$rc = 0;
    }

    return($rc);
}


sub tfr_restore_upgrade_pathto_useradd_cmd
{
    my $useradd_cmd = $EMPTY_STR;

    if ($RTI) {
	$useradd_cmd = File::Spec->catdir($RTI_BINDIR, 'rtiuser.pl');
    }
    if ($DAISY) {
	$useradd_cmd = File::Spec->catdir($DAISY_BINDIR, 'dsyuser.pl');
    }

    if ($useradd_cmd) {
	if ($TEST_RESTORE_UPGRADE_ADD_USERS) {
	    $useradd_cmd = "echo $useradd_cmd";
	}
    }

    return($useradd_cmd);
}


sub tfr_restore_upgrade_pathto_mv_cmd
{
    my $mv_cmd = '/bin/mv';

    if ($TEST_RESTORE_UPGRADE_ADJUST_USERS) {
	$mv_cmd = "echo $mv_cmd";
    }

    return($mv_cmd);
}


sub tfr_restore_upgrade_parse_users
{
    my ($users_listing_file) = @_;

    my %users_tab = ();

    if (open(my $uf_fh, '<', $users_listing_file)) {
	while (my $line = <$uf_fh>) {
	    next if ($line =~ /^\s*$/);
	    next if ($line =~ /^\s*#/);

	    if ($line =~ /^(\S+)\s/) {
		$users_tab{$1} = 1;
	    }
	    if ($line =~ /^(\S+)\s.*RTI Admin/) {
		$users_tab{$1} = 2;
	    }
	    if ($line =~ /^(\S+)\s.*Daisy Admin/) {
		$users_tab{$1} = 2;
	    }
	}
	close($uf_fh);
    }
    else {
	showerror("[restore upgrade] could not open users listing file: $users_listing_file");
    }

    return(%users_tab);
}


#
# Read the shadow file from a backup which
# consists of the lines from the shadow file for each
# user on the system at the time of the backup.
#
# Returns
#   hash with the username as the key and the line as the value.
#
sub tfr_restore_upgrade_parse_shadow_file
{
    my ($shadow_file) = @_;

    my %usersinfo_tab = ();
    my $line;

    if (open(my $ui_fh, '<', $shadow_file)) {
	while ($line = <$ui_fh>) {
	    my $i = index($line, ":");
	    my $username = substr($line, 0, $i);
	    $usersinfo_tab{$username} = $line;
	}
	close($ui_fh);
    }
    else {
	logerror("[restore upgrade] could not open shadow file: $shadow_file");
    }

    return(%usersinfo_tab);
}


sub tfr_restore_upgrade_add_users
{
    my ($users_listing_file) = @_;

    my $rc = 1;

    my $useradd_cmd = tfr_restore_upgrade_pathto_useradd_cmd();
    if ($useradd_cmd) {
	my %users_tab = tfr_restore_upgrade_parse_users($users_listing_file);
	if (scalar(keys(%users_tab))) {
	    foreach my $key (keys(%users_tab)) {
		loginfo("restoring POS user $key...");
		system("$useradd_cmd --add $key 2>> $LOGFILE");
		if ($users_tab{$key} == 2) {
		    loginfo("restoring POS admin $key...");
		    system("$useradd_cmd --enable-admin $key password 2>> $LOGFILE");
		}
	    }
	    loginfo("[restore upgrade] users info restored");
	}
	else {
	    showerror("[restore upgrade] users listing file empty: $users_listing_file");
	    $rc = 0;
	}
    }
    else {
	showerror("[restore upgrade] can't happen: could not determine user add command");
	$rc = 0;
    }

    return($rc);
}


sub tfr_restore_upgrade_adjust_users
{
    my ($users_shadow_file, $system_shadow_file) = @_;

    my $rc = 1;

    my %usersinfo_tab = tfr_restore_upgrade_parse_shadow_file($users_shadow_file);

    my $new_shadow_file = $system_shadow_file . ".$$";

    if (open(my $old_fh, '<', $system_shadow_file)) {
	if (open(my $new_fh, '>', $new_shadow_file)) {
	    while (my $line = <$old_fh>) {
		my $i = index($line, ":");
		my $username = substr($line, 0, $i);
		if (defined($usersinfo_tab{$username})) {
		    print {$new_fh} "$usersinfo_tab{$username}";
		}
		else {
		    print {$new_fh} "$line";
		}
	    }
	    close($new_fh);
	}
	else {
	    logerror("[restore upgrade] could not open new shadow file: $new_shadow_file");
	    $rc = 0;
	}
	close($old_fh);
    }
    else {
	logerror("[restore upgrade] could not open system shadow file: $system_shadow_file");
	$rc = 0;
    }

    if (-e $new_shadow_file && -s $new_shadow_file) {
	system("chmod --reference=$system_shadow_file $new_shadow_file");
	system("chown --reference=$system_shadow_file $new_shadow_file");
	my $mv_cmd = tfr_restore_upgrade_pathto_mv_cmd();
	system("$mv_cmd $new_shadow_file $system_shadow_file");

	loginfo("[restore upgrade] shadow file info restored");
    }
    else {
	if (-e $new_shadow_file) {
	    unlink $new_shadow_file;
	    logerror("[restore upgrade] zero length new shadow file removed: $new_shadow_file");
	    $rc = 0;
	}
    }

    return($rc);
}


#
# Rebuild users info from restored passwd and shadow files.
#
# 1) add new users from users info file
# 2) adjust the shadow file entries for new users
#
sub tfr_restore_upgrade_rebuild_users_info
{
    my ($users_listing_file, $users_shadow_file) = @_;

    my $rc = 1;

    if (tfr_restore_upgrade_add_users($users_listing_file)) {
	loginfo("[restore upgrade] added users from: $users_listing_file");
	my $system_shadow_file = '/etc/shadow';
	if (tfr_restore_upgrade_adjust_users($users_shadow_file, $system_shadow_file)) {
	    loginfo("[restore upgrade] shadow file adjusted for: $users_listing_file");
	}
	else {
	    showerror("[restore upgrade] could not adjust shadow file for: $users_listing_file");
	    $rc = 0;
	}
    }
    else {
	showerror("[restore upgrade] could not add users from: $users_listing_file");
	$rc = 0;
    }

    return($rc);
}


sub tfr_restore_upgrade_homedir_owners
{
    my ($users_listing_file) = @_;

    my $rc = 1;

    my $cmd_prefix = ($TEST_RESTORE_UPGRADE_HOMEDIR_OWNERS) ? 'echo' : $EMPTY_STR;

    my %users_tab = tfr_restore_upgrade_parse_users($users_listing_file);
    if (scalar(keys(%users_tab))) {
	foreach my $key (keys(%users_tab)) {
	    my $homedir = File::Spec->catdir('/home', $key);
	    if (-d $homedir) {
		my $owner = $key . ':';
		my $group = ($RTI) ? 'rti' : 'daisy';
		system("$cmd_prefix chown -R $owner $homedir");
		# some fixups
		system("$cmd_prefix chown 'root:root' $homedir/.bash_logout");
		system("$cmd_prefix chown 'tfsupport:$group' $homedir/.bash_profile");
		loginfo("[restore upgrade] home directory owner/group set: $homedir");
	    } 
	}
    }

    return($rc);
}


#
# Restore user accounts related to the POS.
#
# 1) rebuild users info in passwd and shadow file
# 2) set the perms on user's home directory
#
sub tfr_restore_upgrade_restore_users
{
    my ($users_listing_file, $users_shadow_file) = @_;

    my $rc = 0;

    if (tfr_restore_upgrade_rebuild_users_info($users_listing_file, $users_shadow_file)) {
	if (tfr_restore_upgrade_homedir_owners($users_listing_file)) {
	    $rc = 1;
	}
    }

    return($rc);
}


############################################
##                                        ##
## subsection: file perms                 ##
##                                        ##
############################################

#
# given a list of files, if the read bit for user is NOT set,
# set it to read.
#
# Returns
#   list of filenames changed
#   empty list means none changed
#
sub tfr_file_perms_set_readable
{
    my ($files) = @_;

    my @changed_files = ();

    foreach my $file (@{$files}) {
	if (-e $file) {
	    # get current perms
	    my $sb = File::stat::stat($file);
	    my $file_perms = $sb->mode & oct(7777);

	    # if user's access not readable, make it so
	    unless ($file_perms & oct(400)) {
		$file_perms |= oct(400);
		my $result = chmod $file_perms, $file;
		if ($result == 1) {
		    push(@changed_files, $file);
		}
		else {
		    logerror("[set readable] could not set user access to read: $file");
		}
	    }
	}
	else {
	    loginfo("[set readable] file does not exist: $file");
	}
    }

    return(@changed_files);
}

#
# given a list of files which originally had the read bit
# for user access clear, clear the read bit.
#
# Returns
#   1 for success
#   0 for error and log a message
#
sub tfr_file_perms_unset_readable
{
    my ($changed_files) = @_;

    my $rc = 1;

    foreach my $file (@{$changed_files}) {
	if (-e $file) {
	    # get current perms
	    my $sb = File::stat::stat($file);
	    my $file_perms = $sb->mode & oct(7777);

	    # remove user's read access
	    $file_perms &= ~oct(400);
	    my $result = chmod $file_perms, $file;
	    if ($result != 1) {
		logerror("[unset readable] could not set user access to unreadable: $file");
		$rc = 0;
	    }
	}
	else {
	    loginfo("[unset readable] file does not exist: $file");
	}
    }

    return($rc);
}


############################################
##                                        ##
## subsection: rsync stats api            ##
##                                        ##
############################################

sub tfr_format_rsync_stats
{
    my ($bytes) = @_;

    my $rc = $EMPTY_STR;

    foreach ('B', 'KB', 'MB', 'GB', 'TB', 'PB') {
	if ($bytes < 1024) {
	    $rc = sprintf("%.2f", $bytes) . $_;
	    last;
	}
	$bytes /= 1024;
    }

    return($rc);
}


#
# parse out the totals line output by rsync
#
# Returns
#   list of bytes sent, bytes received, rate on success
#   empty list otherwise
#
sub tfr_parse_log_rsync_stats
{
    my ($rsync_stats_log) = @_;

    my @rc = ();

    # this regex is for the totals line output by rsync
    my $re_sent     = qr{ sent     \s+ (\S+) \s+ bytes \s+  }x;
    my $re_received = qr{ received \s+ (\S+) \s+ bytes \s+  }x;
    my $re_rate     = qr{              (\S+) \s+ bytes\/sec }x;
    my $re          = qr{ $re_sent $re_received $re_rate }x;

    if (open(my $rrlfh, '<', $rsync_stats_log)) {
	while (my $line=<$rrlfh>) {
	    chomp($line);

# perlcritic -3 did not like this line
#	    if ($line =~ m/^sent     \s+ (\S+) \s+ bytes \s+
#                            received \s+ (\S+) \s+ bytes \s+
#                                         (\S+) \s+ bytes\/sec/x) {

	    if ($line =~ m/ $re /x) {
		@rc = ( $1, $2, $3 );
	    }
	}
	close($rrlfh);
    }
    else {
	logerror("could not open rsync stats log: $rsync_stats_log");
    }

    return(@rc);
}


sub tfr_convert_rsync_stats
{
    my ($rsync_stat_raw) = @_;

    my $rsync_stat = 0;

    if ($rsync_stat_raw =~ m/([\d.]+) (K|M|G)*/x) {
	$rsync_stat = $1;
	my $modifier = ($2) ? $2 : 0;
	$rsync_stat *= 1024 if ($modifier eq 'K');
	$rsync_stat *= (1024 * 1024) if ($modifier eq 'M');
	$rsync_stat *= (1024 * 1024 * 1024) if ($modifier eq 'G');
    }

    return($rsync_stat);
}


#
# process the rsync stats file
#
# - get bytes sent, received, and rate
# - add to running total
#
# Returns
#   1 on success
#   0 or error
#
sub tfr_record_rsync_stats
{
    my ($rsync_stats_log) = @_;

    my $rc = 1;


    my @results = tfr_parse_log_rsync_stats($rsync_stats_log);
    if (scalar(@results)) {
	my $prev_sent = $RsyncStatsSent;
	my $prev_sent_formatted = tfr_format_rsync_stats($prev_sent);
	loginfo("rsync stats previous total bytes sent: $prev_sent ($prev_sent_formatted)");

	my $current_sent = tfr_convert_rsync_stats($results[0]);
	my $current_sent_formatted = tfr_format_rsync_stats($current_sent);
	loginfo("rsync stats bytes sent: $current_sent ($current_sent_formatted)");

	$RsyncStatsSent += $current_sent;
	$RsyncStatsReceived += tfr_convert_rsync_stats($results[1]);
	$RsyncStatsRate += tfr_convert_rsync_stats($results[2]);

	my $total_sent_formatted = tfr_format_rsync_stats($RsyncStatsSent);
	loginfo("rsync stats current total bytes sent: $RsyncStatsSent ($total_sent_formatted)");

	# total should never regress
	if ($RsyncStatsSent < $prev_sent) {
	    logerror("can't happen: total bytes sent regressed!");
	}
    }
    else {
	logerror("could not parse rsync stats log: $rsync_stats_log");
	$rc = 0;
    }

    return($rc);
}


sub tfr_append_log_rsync_stats
{
    my ($rsync_stats_log, $logfile) = @_;

    loginfo("BEGIN append rsync stats log") if ($DEBUGMODE);
    if (open(my $infh, '<', $rsync_stats_log)) {
	if (open(my $outfh, '>>', $logfile)) {
	    while (my $line = <$infh>) {
		print {$outfh} $line;
	    }
	    close($outfh);
	}
	close($infh);
    }
    loginfo("END append rsync stats log") if ($DEBUGMODE);

    return(1);
}


sub tfr_init_rsync_stats
{
    $RsyncStatsSent = 0;
    $RsyncStatsReceived = 0;
    $RsyncStatsRate = 0;

    return(1);
}


# take care of the rsync stats
sub tfr_manage_rsync_stats
{
    my ($rsync_stats_path, $rc) = @_;

    # if exit status from rsync was zero, ie no errors or
    # if 24, ie files vanished, then process the rsync stats file
    if ( ($rc == 0) || ($rc == $RSYNC_EXIT_STATUS_VANISHED) ) {
	if (tfr_record_rsync_stats($rsync_stats_path)) {
	    loginfo("rsync stats recorded from: $rsync_stats_path");
	}
	else {
	    logerror("could not record rsync stats from: $rsync_stats_path");
	}
    }

    # now append the rsync stats file to the full logfile
    if (tfr_append_log_rsync_stats($rsync_stats_path, $LOGFILE)) {
	loginfo("rsync stats appended to logfile from temp file");
    }
    else {
	logerror("could not append rsync stats to logfile");
    }

    return($rc);
}


sub tfr_backup_command_status_classify
{
    my ($rc) = @_;

    # now examine the exit status from perl system builtin
    if ($rc == -1) {
	showerror("rsync command failed to execute: $!");
	$rc = $EXIT_COULD_NOT_EXECUTE;
    }
    elsif ($rc & 127) {
	my $signo = ($rc & 127);
	showerror("rsync command died from signal: $signo");
	$rc = $EXIT_FROM_SIGNAL;
    }
    else {
	$rc = ($rc >> 8);

	# exit status could be adjusted when testing is in effect
	$rc = tfr_rsync_exit_status_adjust($rc);

	# classify the error was warning, error, or OK
	$rc = tfr_rsync_exit_status_classify($rc);
    }

    return($rc);
}


#
# Do an rsync of a source tree to a destination
#
# To rsync from local backup dir to backup server:
#   cd /; sudo rsync -avR --delete --rsync-path='sudo rsync' \
#	etc/inittab tfrsync@192.168.2.2:/d/tfrsync
#
# Returns
#   $EXIT_OK for success
#   $EXIT_MOUNT_ERROR if doing device backup and can not mount device
#   $EXIT_BACKUP_TYPE if backup device or remote server not specified
#   > 0 is the rsync exit value
#   
sub tfr_backup_transaction
{
    my ($tobackup, $excludes, $bu_type) = @_;

    # verify list of src paths
    my $file_line_nr = __LINE__;
    if ($#{$tobackup} < 0) {
	loginfo("empty list of paths in \$tobackup near line nr: $file_line_nr");
	return($EXIT_OK);
    }

    my $rc = 0;

    # form prefix for destination argument if possible
    my $dst_prefix = "";

    # destination class used in log message
    my $transaction_class = "";

    # backing up to local device
    if ($DEVICE) {
	unmount_device();
	if (mount_device("rw")) {
	    logerror("Could not mount filesystem on: $DEVICE");
	    return($EXIT_MOUNT_ERROR);
	}
	$dst_prefix = $MOUNTPOINT;
	$transaction_class = "device";
    }

    # backing up to rsync server
    elsif ($SERVER) {
	$transaction_class = "server";
	$dst_prefix = $RSYNC_ACCOUNT . "@" . $RsyncServer . ":";

	# if backup type is a special backup type, ie, one that
	# does not get copied "in place", ie at the same path
	# on the backup server as it is on the primary server,
	# then destination is the tfrsync backup directory.

	if (grep {/$bu_type/} @SPECIAL_BACKUP_TYPES) {
	    if ($RTI) {
		$dst_prefix .= ($RsyncDir) ? $RsyncDir : $DEF_RTI_RSYNC_BU_DIR;
	    }
	    if ($DAISY) {
		$dst_prefix .= ($RsyncDir) ? $RsyncDir : $DEF_DAISY_RSYNC_BU_DIR;
	    }
	}
	else {
	    $dst_prefix .= ($RsyncDir) ? $RsyncDir : '/';
	}
    }

    elsif ($RsyncDir) {
	$dst_prefix = $RsyncDir;
	$transaction_class = "file system";
    }

    elsif ($CLOUD) {
	$dst_prefix = $RSYNC_ACCOUNT . "@" . $CLOUD_SERVER . ":";
	$transaction_class = "cloud";
    }

    else {
	logerror("Backup destination unspecified");
	return($EXIT_BACKUP_TYPE);
    }

    # deal with entities that must be excluded from a "usr2" backup,
    # eg, "doveserver.pl".  When backing up to a rsync server, they
    # can't be backed up in place.
    #
    # To keep them from being backed up in place, they will be
    # put on the global exclude list while backing up "usr2" and
    # removed from the exclude list after backup is complete.
    my $rti_exclusion_push_count = 0;
    if ($bu_type eq $BU_TYPE_USR2) {
	for my $idx (0 .. $#USR2_EXCLUSIONS) {
	    my $exclude_it = 0;
	    if ($SERVER && ($USR2_EXCLUSIONS[$idx][0] = $DEVTYPE_SERVER)) {
		$exclude_it++;
	    }
	    elsif ($CLOUD && ($USR2_EXCLUSIONS[$idx][0] = $DEVTYPE_CLOUD)) {
		$exclude_it++;
	    }
	    elsif ($DEVICE && ($USR2_EXCLUSIONS[$idx][0] = $DEVTYPE_DEVICE)) {
		$exclude_it++;
	    }
	    else {
		if ($USR2_EXCLUSIONS[$idx][0] = $DEVTYPE_ANY) {
		    $exclude_it++;
		}
	    }
	    if ($exclude_it) {
		$rti_exclusion_push_count++;
		push(@{$excludes}, $USR2_EXCLUSIONS[$idx][1]);
		loginfo("element pushed onto excludes list: $USR2_EXCLUSIONS[$idx][1]");
	    }
	}
    }
    my $daisy_exclusion_push_count = 0;
    if ($bu_type eq $BU_TYPE_DAISY) {
	# must exclude an old directory that causes rsync error
	my $daisy_old_edir = "/d/edirectories/2013may/xml";
	if (-d $daisy_old_edir) {
	    push(@{$excludes}, $daisy_old_edir);
	    loginfo("element pushed onto excludes list: $daisy_old_edir");
	    $daisy_exclusion_push_count++;
	}
	if ($SERVER) {
	    # just like for RTI, must exclude "/d/tfrsync"
	    push(@{$excludes}, $DEF_DAISY_RSYNC_BU_DIR);
	    loginfo("element pushed onto excludes list: $DEF_DAISY_RSYNC_BU_DIR");
	    $daisy_exclusion_push_count++;
	}
    }

    # write the list of the exclude files to a temp file IF the
    # backup type is not user specified files - it doesn't make sense to
    # have exceptions for user specified files.
    my $tfn = "";
    if ($bu_type ne $BU_TYPE_USER_FILES) {
	my $exclude_prefix = "";
	$tfn = tfr_rsync_exclude_file($excludes, $exclude_prefix);
    }

    # save the POS users info to the users info dir
    if ($bu_type eq $BU_TYPE_POS_USERS_INFO) {
	if (tfr_save_users_info()) {
	    loginfo("users info saved");
	}
	else {
	    logerror("could not save users info");
	    return($EXIT_USERS_INFO_SAVE);
	}
    }

    # save the pserver info to the pserver info dir
    if ($bu_type eq $BU_TYPE_PSERVER_INFO) {
	if (tfr_save_pserver_info()) {
	    loginfo("pserver info saved");
	}
	else {
	    logerror("could not save pserver info");
	    return($EXIT_PSERVER_INFO_SAVE);
	}
    }

    # save the pserver cloistered files to the pserver cloister dir
    if ($bu_type eq $BU_TYPE_PSERVER_CLOISTER) {
	if (tfr_save_pserver_cloister_files()) {
	    loginfo("pserver cloister files saved");
	}
	else {
	    logerror("could not save the pserver cloister files");
	    return($EXIT_PSERVER_CLOISTER_FILES_SAVE);
	}
    }

    # make any unreadable files from "userconfigs" readable
    my @changed_files = ();
    if (($bu_type eq $BU_TYPE_USER_CONFIGS) && ($OS eq "RHEL6")) {
	@changed_files = tfr_file_perms_set_readable($tobackup);
    }

    loginfo("BEGIN backup transaction");

    if ($DEBUGMODE) {
	tfr_log_rsync_transaction($tobackup, $excludes, $bu_type, $transaction_class);
    }
    else {
	loginfo("backup type: $bu_type");
	loginfo("backup class: $transaction_class");
	loginfo("backup set: @{$tobackup}");
	loginfo("backup exclusions: @{$excludes}");
    }


    ###################################
    # start forming the rsync command #
    ###################################

    # will it be "niced" or not?
    my $rsync_bu_cmd = ($RSYNC_NICE) ? "nice " : "";

    $rsync_bu_cmd .= "rsync -ahv";

    # will rsync compression be used?
    $rsync_bu_cmd .= ($RSYNC_COMPRESSION) ? "z" : "";

    # if path to ssh identity file exists, add it to command
    my $ssh_id_path = tfr_pathto_ssh_id_file($RSYNC_ACCOUNT);
    if (-e $ssh_id_path) {
	$rsync_bu_cmd .= " -e \'ssh -i $ssh_id_path";

	# if using ssh tunnel, add it to ssh option
	my $ssh_tunnel_socket_path = tfr_pathto_ssh_tunnel_socket();
	if (-e $ssh_tunnel_socket_path) {
	    $rsync_bu_cmd .= " -o ControlPath=$ssh_tunnel_socket_path";
	}

	$rsync_bu_cmd .= "\'";
    }

    # dry run or not
    if ($RSYNC_TRIAL) {
	$rsync_bu_cmd .= " -n";
    }

    # delete files located in destination that are not in source
    $rsync_bu_cmd .= " --delete";

    # specify exclude list if not empty
    if ($tfn) {
	$rsync_bu_cmd .= " --exclude-from=$tfn";
    }

    # set up the temp dir location, default to local
    # if the rsync options array already has it specified,
    # then do nothing, else add it.
    unless ($CLOUD) {
	unless (grep {/^--temp-dir=/} @RSYNC_OPTIONS) {
	    if ($RTI) {
		unshift(@RSYNC_OPTIONS, "--temp-dir=/usr2");
	    }
	    if ($DAISY) {
		unshift(@RSYNC_OPTIONS, "--temp-dir=/tmp");
	    }
	}
    }

    # add timeout option
    if ($RSYNC_TIMEOUT) {
	$rsync_bu_cmd .= " --timeout=$RSYNC_TIMEOUT";
    }

    # add any extra options specified in config file
    foreach (@RSYNC_OPTIONS) {
	$rsync_bu_cmd .= " $_";
    }

    # commands to run for remote copy
    my $cmd_remote = "cd /; $rsync_bu_cmd --relative";
    if ($SERVER) {
	$cmd_remote .= " --rsync-path=\'sudo rsync\'";
    }

    #
    # Loop through all the paths to backup
    #
    foreach my $src (@{$tobackup}) {

	last if ($SigIntSeen);

	my $src_remote = $src;
	$src_remote =~ s/^\///;

	# source and destination arguments
	my $rsync_cmd_remote = $cmd_remote;
	$rsync_cmd_remote .= " $src_remote";
	$rsync_cmd_remote .= " $dst_prefix";

	# exec the remote command
	if ($DRY_RUN) {
	    system("echo \"$rsync_cmd_remote\"");
	}
	else {
	    loginfo("backup command: $rsync_cmd_remote");

	    # always init saved rsync exit status
	    tfr_rsync_exit_status_clear();

	    # make a temp file for the rsync stats
	    my ($rsfh, $rsfn) = tempfile($RSYNCSTATS_TEMPLATE, DIR => '/tmp');
	    loginfo("rsync stats redirected to temp file: $rsfn");

	    # exec the rsync command, display output on stdout in addition
	    # to saving output to a log file for further processing
	    $rc = system("$rsync_cmd_remote 2>&1 | tee $rsfn ; ( exit \${PIPESTATUS[0]} )");

	    # take care of the rsync stats
	    $rc = tfr_manage_rsync_stats($rsfn, $rc);

	    # remove the rysnc stats temp file
	    if (unlink($rsfn)) {
		loginfo("rsync stats temp file unlinked: $rsfn");
	    }
	    else {
		logerror("could not unlink rsync stats temp file: $rsfn");
	    }

	    # classify backup command results
	    $rc = tfr_backup_command_status_classify($rc);

	    # break on errors
	    if ($rc == 0) {
		loginfo("backup command successful: $rc");
	    }
	    else {
		last;
	    }
	}
    }

    if ($tfn) {
	if (unlink($tfn)) {
	    loginfo("rsync excludes temp file unlinked: $tfn");
	}
	else {
	    logerror("could not unlink rsync excludes temp file: $tfn");
	}
    }

    unless ($DRY_RUN) {
	if ($DEVICE) {
	    unmount_device();
	}
    }

    # deal with exclusions...
    # remove them from the global exclude list now
    # that the transaction is over.
    if ($bu_type eq $BU_TYPE_USR2) {
	while ($rti_exclusion_push_count--) {
	    pop(@{$excludes});
	}
    }
    if ($bu_type eq $BU_TYPE_DAISY) {
	while ($daisy_exclusion_push_count--) {
	    pop(@{$excludes});
	}
    }

    # restore file perms for some userconfig files
    if (($bu_type eq $BU_TYPE_USER_CONFIGS) && ($OS eq "RHEL6")) {
	if (@changed_files) {
	    if (tfr_file_perms_set_unreadable(\@changed_files)) {
		loginfo("userconfig file perms restored");
	    }
	    else {
		logerror("could not restore userconfig file perms");
	    }
	}
    }

    loginfo("END backup transaction");

    return($rc);
}


#
# Given a backup type, return a list of files to backup.
#
sub tfr_backup_file_list
{
    my ($bu_type) = @_;

    my @bu_type_files = ();

    if ($bu_type eq $BU_TYPE_USER_CONFIGS) {
	my @possible_files = qw(
	    /etc/group
	    /etc/group-
	    /etc/gshadow
	    /etc/gshadow-
	    /etc/login.defs
	    /etc/passwd
	    /etc/passwd-
	    /etc/shadow
	    /etc/shadow-
	    /etc/sudoers
	    /etc/pam.d
	    /home
	    /root
	);

	if ($OS eq "RHEL5") {
	    push(@possible_files, "/var/log/faillog");
	}
	if ($OS eq "RHEL6") {
	    push(@possible_files, "/var/log/tallylog");
	}
	foreach (@possible_files) {
	    if (-e $_) {
		push(@bu_type_files, $_);
	    }
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_PRINT_CONFIGS) {
	@bu_type_files = qw(
	    /etc/printcap
	    /etc/cups
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_OS_CONFIGS) {
	my @common_files = qw(
	    /etc/inittab
	    /etc/log.d
	    /etc/mail
	    /etc/redhat-release
	    /etc/samba
	    /etc/yum
	    /etc/yum.conf
	    /etc/yum.repos.d
	);

	# contains setting for kernel log message priority for console
	if ($OS eq "RHEL6") {
	    push(@common_files, "/etc/rsyslog.conf");
	}
	if ($OS eq "RHEL5") {
	    push(@common_files, "/etc/sysconfig/syslog");
	}

	# check for optional files or directories
	my @possible_files = (
	    '/etc/httpd',           # web server config directory
	    '/usr/src/redhat'       # source directory
	);
	foreach (@possible_files) {
	    if (-e $_) {
		push(@common_files, $_);
	    }
	}

	foreach (@common_files) {
	    if (-e $_) {
		push(@bu_type_files, $_);
	    }
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_NET_CONFIGS) {
	@bu_type_files = qw(
	    /etc/hosts
	    /etc/hosts.allow
	    /etc/hosts.deny
	    /etc/resolv.conf
	    /etc/ssh
	    /etc/sysconfig/iptables
	    /etc/sysconfig/network
	    /etc/sysconfig/networking/profiles
	);

	my @ifcfg_files = glob("/etc/sysconfig/network-scripts/ifcfg*");
	foreach (@ifcfg_files) {
	    push(@bu_type_files, $_);
	}

	# this path does not exist on all platforms
	@ifcfg_files = glob("/etc/sysconfig/networking/devices/ifcfg*");
	foreach (@ifcfg_files) {
	    push(@bu_type_files, $_);
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_RTI_CONFIGS) {
	@bu_type_files = (
	    "/etc/sysconfig/i18n",
	    "/etc/profile",
	    "/etc/profile.d/rti.sh",
	    "/etc/rc.d/init.d/rti",
	    "/etc/rc.d/init.d/blm",
	    "/etc/rc.d/init.d/bbj",
	    "/etc/services",
	    "/etc/mime.types",
	    "$RTIDIR/config",
	    "/var/spool/fax",
	    "/usr/local/lib/BITMAPS",
	    "/usr/java",
	);

	# check for optional files or directories
	my @optional_files = (
	    '/etc/profile.d/pro5.sh',
	);
	foreach (@optional_files) {
	    if (-e $_) {
		push(@bu_type_files, $_);
	    }
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_DAISY_CONFIGS) {
	@bu_type_files = (
	    "/etc/inittab",
	    "/etc/profile.d/daisy.sh",
	    "$DAISYDIR/config",
	);

	if ($OS eq "RHEL5") {
	    # a long time ago this file was standard on all
	    # Daisy RHEL5 systems and should be present, but
	    # it seems that bit rot has set in and it is not
	    # present on all Daisy servers, so skip it if it
	    # does not exist
	    my $daisy_init_file = "/etc/rc.d/init.d/daisy";
	    if (-f $daisy_init_file) {
		push(@bu_type_files, $daisy_init_file);
	    }
	    else {
		loginfo("missing Daisy init file ignored: $daisy_init_file");
	    }
	}
	if ($OS eq "RHEL6") {
	    push(@bu_type_files, "/etc/rc.d/init.d/zeedaisy");
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_USR2) {
	@bu_type_files = (
	    "/usr2",
	);

	# additional directories for RTI version 14
	my @rti14_files = qw(
	    /var/www
	);

	foreach (@rti14_files) {
	    if (-d $_) {
		push(@bu_type_files, $_);
	    }
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_BBXD) {
	@bu_type_files = (
	    "/usr2/bbx/bbxd",
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_BBXPS) {
	@bu_type_files = (
	    "/usr2/bbx/bbxps",
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_DAISY) {
	@bu_type_files = (
	    "/d",
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_LOG_FILES) {
	@bu_type_files = (
	    "/var/log",
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_USER_FILES) {
	@bu_type_files = @USERFILES;

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_SINGLEFILE) {
	if (@SinglefilesCLO) {
	    @bu_type_files = @SinglefilesCLO;
	}

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_PSERVER_CLOISTER) {
	my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
	@bu_type_files = (
	    $pserver_cloister_dir_path,
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_POS_USERS_INFO) {
	my $users_info_bu_dir = tfrm_pathto_users_info_dir();
	@bu_type_files = (
	    $users_info_bu_dir,
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_PSERVER_INFO) {
	my $pserver_infodir = tfrm_pathto_pserver_info_dir();
	@bu_type_files = (
	    $pserver_infodir,
	);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_POS_LOG_FILES) {
	@bu_type_files = ( $RTI_LOGDIR ) if ($RTI);
	@bu_type_files = ( $DAISY_LOGDIR) if ($DAISY);

	return(@bu_type_files);
    }

    if ($bu_type eq $BU_TYPE_POS_SUMMARY_LOG) {
	@bu_type_files = ( $SummaryLogfile );

	return(@bu_type_files);
    }

    logerror("[tfr_backup_file_list()] Can't happen: \$bu_type = $bu_type");

    return(@bu_type_files);
}


#
# Save files associated with a backup type
#
sub tfr_backup_by_type
{
    my ($bu_type, $excludes) = @_;

    my $rc = 0;

    my @toback = tfr_backup_file_list($bu_type);
    if (@toback) {
	if ($DEVICE eq $DEVTYPE_SHOW_ONLY || $SHOW_ONLY) {
	    foreach (@toback) {
		print "$_\n";
	    }
	}
	else {
	    showinfo("BEGIN backup type: $bu_type");

	    $rc = tfr_backup_transaction(\@toback, $excludes, $bu_type);

	    loginfo("END backup type: $bu_type");
	}
    }
    else {
	loginfo("Empty list of files for backup type: $bu_type");
    }

    return($rc);
}


#
# top level sub for "--backup"
#
# Backup Files.
#
# The backup of all backup types funnels through this function.
#
sub tfr_backup_files
{
    my ($backup, $excludes) = @_;

    my $returnval = 0;

    # Verify the backup device is accessible via mount
    if ($DEVICE eq $DEVTYPE_BLOCK || $DEVICE eq $DEVTYPE_IMAGE) {
	mount_device("rw");
	if (is_rsync_bu_device_mounted()) {
	    unmount_device();
	}
	else {
	    showerror("Could not verify backup device is accessible via mount: $DEVICE");
	    return($EXIT_MOUNT_ERROR);
	}
    }

    my @types_to_backup = (grep {/^all$/} @{$backup}) ? @BACKUP_EQUALS_ALL_BACKUP_TYPES : @{$backup};

    if ($CLOUD || $SERVER) {
	my $server_addr = ($SERVER) ? $RsyncServer : $CLOUD_SERVER;
	if (tfr_open_ssh_tunnel($RSYNC_ACCOUNT, $server_addr)) {
	    loginfo("ssh tunnel socket opened for backup: $RSYNC_ACCOUNT, $server_addr");
	}
	else {
	    showerror("could not open ssh tunnel for backup: $RSYNC_ACCOUNT, $server_addr");
	    return($EXIT_SSH_TUNNEL_OPEN);
	}
    }

    #
    # normally, only one backup iteration is attempted.
    #
    my $max_backup_iterations = 1;

    #
    # If backup retries are specified, then additional attempts
    # are performed.
    #
    my $retry_backup_reps = tfr_retry_backup_reps();
    $max_backup_iterations += $retry_backup_reps;

    #
    # And if backup wait is specified, there will be some
    # time between each try.
    #
    my $retry_backup_wait = tfr_retry_backup_wait();

    for (my $i=0; $i < $max_backup_iterations; $i++) {

	foreach my $bu_type (@types_to_backup) {

	    last if ($SigIntSeen);

	    next if ( ($bu_type eq $BU_TYPE_USR2) && ($RTI == 0) );
	    next if ( ($bu_type eq $BU_TYPE_RTI_CONFIGS) && ($RTI == 0) );
	    next if ( ($bu_type eq $BU_TYPE_DAISY) && ($DAISY == 0) );
	    next if ( ($bu_type eq $BU_TYPE_DAISY_CONFIGS) && ($DAISY == 0) );

	    $returnval = tfr_backup_by_type($bu_type, $excludes);
	    if ($returnval != 0) {
		logerror("could not backup type: $bu_type");
		last;
	    }
	}

	# record and log number of retries
	# NOTE: $i will only be > 0 only if backup retries are enabled and
	# the first attempt was already completed.
	if ($i > 0) {
	    tfr_retry_backup_record_retries($i);
	    loginfo("[retrying] backup retries completed: $i");
	}

	$returnval = tfr_rsync_exit_status_prepare($returnval);

	if ($returnval == $EXIT_RSYNC_ERROR) {
	    my $rsync_exit_status = tfr_rsync_exit_status_fetch();
	    if ($rsync_exit_status == $RSYNC_EXIT_STATUS_IO_ERROR) {
		loginfo("[retrying] rsync reported I/O error exit status: $RSYNC_EXIT_STATUS_IO_ERROR");
	    }
	    elsif ($rsync_exit_status == $RSYNC_EXIT_STATUS_SSH_ERROR) {
		loginfo("[retrying] rsync reported SSH error exit status: $RSYNC_EXIT_STATUS_SSH_ERROR");
	    }
	    else {
		# exit loop if the rsync error was ordinary error
		loginfo("[retrying] no retry for ordinary rsync error: $rsync_exit_status");
		last;
	    }
	}
	else {
	    # exit loop if not an rsync error
	    loginfo("[retrying] no retry for non-rsync error: $returnval");
	    last;
	}

	#
	# sleep unless on the last iteration.
	#
	if ( ($i < ($max_backup_iterations-1)) && ($retry_backup_wait) ) {
	    sleep($retry_backup_wait);
	    loginfo("[retrying] time waited in seconds: $retry_backup_wait");
	}
    }

    if ($CLOUD || $SERVER) {
	my $server_addr = ($SERVER) ? $RsyncServer : $CLOUD_SERVER;
	if (tfr_close_ssh_tunnel($RSYNC_ACCOUNT, $server_addr)) {
	    loginfo("ssh tunnel socket for backup closed: $RSYNC_ACCOUNT, $server_addr");
	}
	else {
	    showerror("could not close ssh tunnel socket for backup: $RSYNC_ACCOUNT, $server_addr");
	    if ($returnval == 0) {
		$returnval = $EXIT_SSH_TUNNEL_CLOSE;
	    }
	}
    }

    return($returnval);
}


#
# Do an "list only" rsync of a source tree to a destination
#
# To rsync from local path to local backup dir:
#   cd /; sudo rsync -avR --delete home /d/backup/
#
# To rsync from local backup dir to backup server:
#   cd /; sudo rsync -avR --delete --rsync-path='sudo rsync' \
#	etc/inittab tfrsync@192.168.2.2:/d/backup
#
# Returns
#   0 for success
#   -1 if list of items to backup is empty
#   -2 if doing device backup and can not mount device
#   -3 if backup device or remote server not specified
#   > 0 is the rsync exit value
#   
sub tfr_list_only_rsync_transaction
{
    my ($tobackup, $excludes, $bu_type) = @_;

    my $rc = 0;

    loginfo("##### BEGIN list transaction");

    # form prefix for destination argument if possible
    my $dst_prefix = "";

    # listing files from a local device
    if ($DEVICE) {
	$dst_prefix = $MOUNTPOINT;
    }

    # listing files from an rsync server
    elsif ($SERVER) {
	$dst_prefix = $RSYNC_ACCOUNT . "@" . $RsyncServer . ":";
	if ($RsyncDir) {
	    $dst_prefix .= $RsyncDir;
	}
    }

    elsif ($RsyncDir) {
	$dst_prefix = $RsyncDir;
    }

    else {
	logerror("[tfr_list_only_rsync_transaction()] source unspecified");
	return(-3);
    }

    # write the list of the exclude files to a temp file IF the
    # backup type is not user specified files - it doesn't make sense to
    # have exceptions for user specified files.
    my $tfn = "";
    if ($bu_type ne "userfiles") {
	my $exclude_prefix = "";
	$tfn = tfr_rsync_exclude_file($excludes, $exclude_prefix);
    }

    # form the rsync command
    my $cmd = "nice rsync -av";

    # if path to ssh identity file exists, add it to command
    my $ssh_id_path = tfr_pathto_ssh_id_file($RSYNC_ACCOUNT);
    if (-e $ssh_id_path) {
	$cmd .= " -e \'ssh -i $ssh_id_path";

	# if using ssh tunnel, add it to ssh option
	my $ssh_tunnel_socket_path = tfr_pathto_ssh_tunnel_socket();
	if (-e $ssh_tunnel_socket_path) {
	    $cmd .= " -o ControlPath=$ssh_tunnel_socket_path";
	}

	$cmd .= "\'";
    }

    # dry run or not
    if ($RSYNC_TRIAL) {
	$cmd .= " -n";
    }

    # specify exclude list if not empty
    if ($tfn) {
	$cmd .= " --exclude-from=$tfn";
    }

    # commands to run for local copy and remote copy
    my $cmd_remote = "cd /; $cmd --relative";
    if ($SERVER) {
	$cmd_remote .= " --rsync-path=\'sudo rsync\'";
    }

    #
    # Loop through all the paths to list
    #
    foreach my $src (@{$tobackup}) {

	my $src_remote = $src;
	$src_remote =~ s/^\///;

	# source and destination arguments
	my $rsync_cmd_remote = $cmd_remote;
	$rsync_cmd_remote .= " --list-only";
	$rsync_cmd_remote .= " $src_remote";
	$rsync_cmd_remote .= " $dst_prefix";

	# exec the remote command
	if ($DRY_RUN) {
	    system("echo \"$rsync_cmd_remote\"");
	}
	else {
	    loginfo("Execing rsync command for remote or device backup:");
	    loginfo($rsync_cmd_remote);
	    $rc = system("$rsync_cmd_remote");
	    last if ($rc != 0);
	}
    }

    unlink($tfn);

    loginfo("##### END list rsync transaction");

    return($rc);
}


sub tfr_list_by_type
{
    my ($bu_type) = @_;

    my $returnval = 0;

    showinfo("BEGIN Listing files for backup type: $bu_type");

    my @tolist = tfr_backup_file_list($bu_type);
    if (@tolist) {
	$returnval = tfr_list_only_rsync_transaction(\@tolist, \@EXCLUDES, $bu_type);
    }
    else {
	loginfo("Empty list of files for backup type: $bu_type");
    }

    loginfo("END Listing files for backup type: $bu_type");

    return($returnval);
}


sub tfr_list_files
{
    my ($list_ref, $bu_device_type) = @_;

    my $returnval = 0;

    if (is_mount_required($bu_device_type)) {
	mount_device("ro");
	unless (is_rsync_bu_device_mounted()) {
	    logerror("error mounting backup device for listing files: $DEVICE");
	    return(-1);
	}
    }

    my @bu_types = @{$list_ref};

    if (grep {/^all$/} @bu_types) {
	@bu_types = (
	    $BU_TYPE_USR2,
	    $BU_TYPE_DAISY,
	    $BU_TYPE_PRINT_CONFIGS,
	    $BU_TYPE_RTI_CONFIGS,
	    $BU_TYPE_DAISY_CONFIGS,
	    $BU_TYPE_OS_CONFIGS,
	    $BU_TYPE_NET_CONFIGS,
	    $BU_TYPE_USER_CONFIGS,
	    $BU_TYPE_LOG_FILES,
	    $BU_TYPE_USER_FILES,
	    $BU_TYPE_PSERVER_CLOISTER,
	);
    }

    if ($SERVER) {
	if (tfr_open_ssh_tunnel($RSYNC_ACCOUNT, $RsyncServer)) {
	    loginfo("ssh tunnel socket for listing opened: $RSYNC_ACCOUNT, $RsyncServer");
	}
	else {
	    showerror("could not open ssh tunnel socket for listing: $RSYNC_ACCOUNT, $RsyncServer");
	    return($EXIT_SSH_TUNNEL_OPEN);
	}
    }

    loginfo("---- BEGIN List Files ----");

    foreach my $bu_type (@bu_types) {

	if ( ($bu_type eq $BU_TYPE_USR2) && ($RTI == 0) ) {
	    next;
	}
	elsif ( ($bu_type eq $BU_TYPE_RTI_CONFIGS) && ($RTI == 0) ) {
	    next;
	}
	elsif ( ($bu_type eq $BU_TYPE_DAISY) && ($DAISY == 0) ) {
	    next;
	}
	elsif ( ($bu_type eq $BU_TYPE_DAISY_CONFIGS) && ($DAISY == 0) ) {
	    next;
	}

	if ($SERVER) {
	    $returnval = tfr_list_by_type($bu_type);
	    if ($returnval != 0) {
		logerror("error listing files for backup type: $bu_type");
		return($returnval);
	    }
	}
	else {
	    my @bu_type_file_list = tfr_backup_file_list($bu_type);
	    foreach (@bu_type_file_list) {
		print "$_\n";
	    }
	}
    }

    if ($SERVER) {
	if (tfr_close_ssh_tunnel($RSYNC_ACCOUNT, $RsyncServer)) {
	    loginfo("ssh tunnel socket for listing closed: $RSYNC_ACCOUNT, $RsyncServer");
	}
	else {
	    showerror("could not close ssh tunnel socket for listing: $RSYNC_ACCOUNT, $RsyncServer");
	    return($EXIT_SSH_TUNNEL_CLOSE);
	}
    }

    if (is_mount_required($bu_device_type)) {
	unmount_device();
    }

    loginfo("---- END List Files ----");

    return(0);
}


sub is_restore_type_compatible
{
    my ($restore_type) = @_;

    my $rc = 0;

    if ( ($restore_type eq $BU_TYPE_USR2) && ($RTI != 1) ) {
	showerror("restore type only compatible with RTI: $restore_type");
    }
    elsif ( ($restore_type eq $BU_TYPE_RTI_CONFIGS) && ($RTI != 1) ) {
	showerror("restore type only compatible with RTI: $restore_type");
    }
    elsif ( ($restore_type eq $BU_TYPE_DAISY) && ($DAISY != 1) ) {
	showerror("restore type only compatible with Daisy: $restore_type");
    }
    elsif ( ($restore_type eq $BU_TYPE_DAISY_CONFIGS) && ($DAISY != 1) ) {
	showerror("restore type only compatible with Daisy: $restore_type");
    }
    elsif ( ($restore_type eq $BU_TYPE_LOG_FILES) && is_restore_in_place() ) {
	showerror("restore type not allowed without alternate location: $restore_type");
    }
    else {
	$rc = 1;
    }

    return($rc);
}


sub tfr_restore_perms
{
    my ($restore_type) = @_;

    my $rc = 1;

    if ($restore_type eq $BU_TYPE_BBXD || $restore_type eq $BU_TYPE_BBXPS) {
	$restore_type = $BU_TYPE_USR2;
    }
    if ($restore_type eq $BU_TYPE_SINGLEFILE) {
	loginfo("[restore perms] perms are not saved for this backup type: $restore_type");
    }
    else {
	my $perm_file_path = tfrm_pathto_perm_file($restore_type);
	if ($perm_file_path) { 
	    print "[restore perms] restoring perms from $perm_file_path...";
	    if (tfr_restore_perms_from_perm_file($perm_file_path)) {
		print "\n";
		loginfo("[restore perms] perms restored from: $perm_file_path");
	    }
	    else {
		logerror("[restore perms] error restoring perms from perm file: $perm_file_path");
		$rc = 0;
	    }
	}
	else {
	    logerror("[restore perms] perm file does not exist for backup type: $restore_type");
	    $rc = 0;
	}
    }

    return($rc);
}


#
# top level sub for "--restore"
#
# Restore some or all files based on what restore type
# was specified on the commandline.  Also restore
# individual files if so specified as command line args.
#
# Returns
#   0 for success
#   !0 for error
#
sub tfr_restore_files
{
    my ($restore_ref, $restore_excludes_ref, $restore_dir) = @_;

    my $rc = 0;

    loginfo("---- BEGIN Restore Files ----");

    # Verify the backup device is accessible via mount
    if ($DEVICE) {
	mount_device("ro");
	if (is_rsync_bu_device_mounted()) {
	    unmount_device();
	}
	else {
	    showerror("[restore files] could not verify backup device is accessible via mount: $DEVICE");
	    return(-1);
	}
    }

    #
    # "restore all" does not include "logfiles" anymore
    #
    my @restore_types = @{$restore_ref};
    if (grep {/^$BU_TYPE_ALL$/} @restore_types) {
	@restore_types = @RESTORE_EQUALS_ALL_BACKUP_TYPES;
    }

    if ($CLOUD || $SERVER) {
	my $server_addr = ($SERVER) ? $RsyncServer : $CLOUD_SERVER;
	if (tfr_open_ssh_tunnel($RSYNC_ACCOUNT, $server_addr)) {
	    loginfo("[restore files] ssh tunnel socket opened: $RSYNC_ACCOUNT, $server_addr");
	}
	else {
	    showerror("[restore files] could not open ssh tunnel: $RSYNC_ACCOUNT, $server_addr");
	    return($EXIT_SSH_TUNNEL_OPEN);
	}
    }

    #
    # before restoring any files, first determine the kind
    # of source platform by restoring the pserver info file;
    # put it into a special directory reserved just for it.
    #
    if ($RestoreUpgradeCLO) {
	# make the pserver info restore dir if necessary
	my $restored_pserver_info_dir_path = tfrm_pathto_restored_pserver_info_dir();
	if (tfr_util_mkdir($restored_pserver_info_dir_path)) {
	    loginfo("[restore files] restored pserver info dir: $restored_pserver_info_dir_path");
	    my @no_excludes = ();
	    if (tfr_restore_by_type($BU_TYPE_PSERVER_INFO, \@no_excludes, $restored_pserver_info_dir_path)) {
		loginfo("[restore files] pserver info file restored to: $restored_pserver_info_dir_path");
	    }
	    else {
		logerror("[restore files] could not restore pserver info file");
	    }
	}
	else {
	    logerror("[restore files] could not make restored pserver info dir: $restored_pserver_info_dir_path");
	}
    }

    foreach my $restore_type (@restore_types) {

	last if ($SigIntSeen);

	if (is_restore_type_compatible($restore_type)) {

	    $rc = tfr_restore_by_type($restore_type, $restore_excludes_ref, $restore_dir);

	    if ($rc != 0) {
		showerror("[restore files] could not restore type: $restore_type");
		last;
	    }
	    else {
		if (is_restore_in_place()) {
		    if (tfr_restore_perms($restore_type)) {
			loginfo("[restore files] perms restored for: $restore_type");
		    }
		    else {
			logerror("[restore files] could not restore perms for: $restore_type");
		    }
		}
	    }
	}
    }

    if ($CLOUD || $SERVER) {
	my $server_addr = ($SERVER) ? $RsyncServer : $CLOUD_SERVER;
	if (tfr_close_ssh_tunnel($RSYNC_ACCOUNT, $server_addr)) {
	    loginfo("[restore files} ssh tunnel socket closed: $RSYNC_ACCOUNT, $server_addr");
	}
	else {
	    showerror("[restore files] could not close ssh tunnel socket: $RSYNC_ACCOUNT, $server_addr");
	    return($EXIT_SSH_TUNNEL_CLOSE);
	}
    }

    #
    # take care of items that need attending to after
    # the restore is finished and doing an upgrade.
    #
    if ($rc == 0) {
	if ($CLOUD && $RestoreUpgradeCLO) {
	    my $users_listing_file = tfrm_pathto_users_listing_file();
	    my $users_shadow_file = tfrm_pathto_users_shadow_file();
	    if (-f $users_listing_file) {
		if (-f $users_shadow_file) {
		    if (tfr_restore_upgrade_restore_users($users_listing_file, $users_shadow_file)) {
			loginfo("[restore files] users restored after upgrade: $users_listing_file");
		    }
		    else {
			showerror("[restore files] could not restore users after upgrade: $users_listing_file");
		    }
		}
		else {
		    showerror("[restore files] file required for restore upgrade does not exist: $users_shadow_file");
		}
	    }
	    else {
		showerror("[restore files] file required for restore upgrade does not exist: $users_listing_file");
	    }
	}
    }

    if ($RestoreUpgradeCLO) {
	my $restored_pserver_info_dir_path = tfrm_pathto_restored_pserver_info_dir();
	if (tfr_util_rmdir($restored_pserver_info_dir_path)) {
	    loginfo("[restore files] restored pserver info dir removed: $restored_pserver_info_dir_path");
	}
	else {
	    logerror("[restore files] could not removed restored pserver info dir: $restored_pserver_info_dir_path");
	}
    }

    loginfo("---- END Restore Files ----");

    return($rc);
}


#
# Call the function that handles the specified restore type.
#
sub tfr_restore_by_type
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = 0;

    # map restore types to a function
    my @ftab = (
	[\&tfr_restore_usr2, $BU_TYPE_USR2],
	[\&tfr_restore_bbxd, $BU_TYPE_BBXD],
	[\&tfr_restore_bbxps, $BU_TYPE_BBXPS],
	[\&tfr_restore_rticonfigs, $BU_TYPE_RTI_CONFIGS],
	[\&tfr_restore_daisy, $BU_TYPE_DAISY],
	[\&tfr_restore_dsyconfigs, $BU_TYPE_DAISY_CONFIGS],
	[\&tfr_restore_userconfigs, $BU_TYPE_USER_CONFIGS],
	[\&tfr_restore_printconfigs, $BU_TYPE_PRINT_CONFIGS],
	[\&tfr_restore_netconfigs, $BU_TYPE_NET_CONFIGS],
	[\&tfr_restore_osconfigs, $BU_TYPE_OS_CONFIGS],
	[\&tfr_restore_userfiles, $BU_TYPE_USER_FILES],
	[\&tfr_restore_logfiles, $BU_TYPE_LOG_FILES],
	[\&tfr_restore_singlefile, $BU_TYPE_SINGLEFILE],
	[\&tfr_restore_specialfiles, $BU_TYPE_PSERVER_CLOISTER],
	[\&tfr_restore_pserver_info, $BU_TYPE_PSERVER_INFO],
    );

    # now call the function appropriate to the restore type
    foreach my $ftab_entry (@ftab) {
	my ($func, $type) = @$ftab_entry;
	if ($restore_type eq $type) {
	    $rc = &$func($restore_type, $restore_excludes, $restore_dir);
	    last;
	}
    }

    if ($rc != 0) {
	logerror("[restore by type] non-zero return code: $rc");
    }

    return($rc);
}


sub tfr_restore_transaction
{
    my ($torestore, $excludes, $bu_type, $restore_dir) = @_;

    # if there are no files to restore, just return
    if (scalar(@{$torestore})) {
	loginfo("[restore transaction] empty list of paths to restore");
	return(0);
    }

    my $rc = 0;

    # form prefix for source argument if possible
    my $src_prefix = "";
    my $exclude_prefix = "";

    my $transaction_class = $EMPTY_STR;

    # restoring from a local device
    if ($DEVICE) {
	if (! $DRY_RUN) {
	    unmount_device();
	    if (mount_device("ro")) {
		logerror("Could not mount filesystem on: $DEVICE");
		return(-2);
	    }
	}
	$src_prefix = $MOUNTPOINT;
	$exclude_prefix = $src_prefix;
	$transaction_class = "device";
    }

    # restoring from a backup server
    elsif ($SERVER) {
	$src_prefix = $RSYNC_ACCOUNT . '@' . $RsyncServer . ':';

	# if backup type is a special backup type, ie, one that
	# does not get copied "in place", ie at the same path
	# on the backup server as it is on the primary server,
	# then source is the tfrsync backup directory.

	if (grep {/$bu_type/} @SPECIAL_BACKUP_TYPES) {
	    if ($RTI) {
		$src_prefix .= ($RsyncDir) ? $RsyncDir : $DEF_RTI_RSYNC_BU_DIR;
	    }
	    if ($DAISY) {
		$src_prefix .= ($RsyncDir) ? $RsyncDir : $DEF_DAISY_RSYNC_BU_DIR;
	    }
	}
	else {
	    $src_prefix .= ($RsyncDir) ? $RsyncDir : '/';
	}
	$transaction_class = "server";
    }

    # restoring from a local file system
    elsif ($RsyncDir) {
	$src_prefix = $RsyncDir;
	$exclude_prefix = $src_prefix;
	$transaction_class = "local fs";
    }

    # restoring from a cloud server
    elsif ($CLOUD) {
	$src_prefix = $RSYNC_ACCOUNT . "@" . $CLOUD_SERVER . ":";
	$transaction_class = "cloud";
    }

    else {
	logerror("Restore source unspecified");
	return(-3);
    }

    loginfo("##### BEGIN restore transaction");

    if ($DEBUGMODE) {
	tfr_log_rsync_transaction($torestore, $excludes, $bu_type, $transaction_class);
    }
    else {
	loginfo("restore type: $bu_type");
	loginfo("restore class: $transaction_class");
	loginfo("restore set: @{$torestore}");
	loginfo("restore exclusions: @{$excludes}");
    }

    # write the list of the exclude files to a temp file IF the
    # backup type is not user specified files - it doesn't make sense to
    # have exceptions for user specified files.
    my $tfn = "";
    if ($bu_type ne $BU_TYPE_USER_FILES) {
	$tfn = tfr_rsync_exclude_file($excludes, $exclude_prefix);
    }

    # start forming the rsync command
    my $rsync_cmd = "rsync -ahv";

    # if path to ssh identity file exists, add it to command
    my $ssh_id_path = tfr_pathto_ssh_id_file($RSYNC_ACCOUNT);
    if (-e $ssh_id_path) {
	$rsync_cmd .= " -e \'ssh -i $ssh_id_path";

	# if using ssh tunnel, add it to ssh option
	my $ssh_tunnel_socket_path = tfr_pathto_ssh_tunnel_socket();
	if (-e $ssh_tunnel_socket_path) {
	    $rsync_cmd .= " -o ControlPath=$ssh_tunnel_socket_path";
	}

	$rsync_cmd .= "\'";
    }

    # dry run or not
    if ($RSYNC_TRIAL) {
	$rsync_cmd .= " -n";
    }

    # delete files located in destination that are not in source
    unless ($CLOUD) {
	$rsync_cmd .= " --delete";
    }

    # specify exclude list if not empty
    if ($tfn) {
	$rsync_cmd .= " --exclude-from=$tfn";
    }

    # set up the temp dir location, default to local
    # if the rsync options array already has it specified,
    # then do nothing, else add it.
    unless ($CLOUD) {
	unless (grep {/^--temp-dir=/} @RSYNC_OPTIONS) {
	    if ($RTI) {
		unshift(@RSYNC_OPTIONS, "--temp-dir=/usr2");
	    }
	    if ($DAISY) {
		unshift(@RSYNC_OPTIONS, "--temp-dir=/tmp");
	    }
	}
    }

    # add timeout option
    if ($RSYNC_TIMEOUT) {
	$rsync_cmd .= " --timeout=$RSYNC_TIMEOUT";
    }

    # add any extra options specified in config file
    foreach (@RSYNC_OPTIONS) {
	$rsync_cmd .= " $_";
    }

    # commands to run
    $rsync_cmd = "cd /; $rsync_cmd --relative";
    #my $cmd_remote = "$cmd --relative";
    #my $cmd_remote = $cmd;
    if ($SERVER) {
	$rsync_cmd .= " --rsync-path=\'sudo rsync\'";
    }

    #
    # Loop through all the paths to restore
    #
    foreach my $src (@{$torestore}) {

	last if ($SigIntSeen);

	my $src_path = $src;
	if ($CLOUD) {
	    $src_path =~ s/^\///;
	}
	#$src_path .= '/';

	# source and destination arguments
	my $cmd = $rsync_cmd;
	$cmd .= " $src_prefix";
	$cmd .= "$src_path";
	my $dst = $restore_dir;
	$cmd .= " $dst";

	# exec the remote command
	if ($DRY_RUN) {
	    system("echo \"$cmd\"");
	}
	else {
	    loginfo("restore command: $cmd");

	    $rc = system("$cmd 2>&1 | tee -a $LOGFILE ; ( exit \${PIPESTATUS[0]} )");

	    # break on errors
	    if ($rc == 0) {
		loginfo("restore command successful: $rc");
	    }
	    else {
		last;
	    }
	}
    }

    if ($tfn) {
	unlink($tfn);
    }

    unless ($DRY_RUN) {
	if ($DEVICE) {
	    unmount_device();
	}
    }

    loginfo("##### END restore rsync transaction");

    return($rc);
}


# Restore RTI /usr2
sub tfr_restore_usr2
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restore RTI /usr2");

    #
    # Only stop RTI if restoring on top of /usr2...
    # which is true only if restoring in place.
    #
    my $clobber_rti = (is_restore_in_place()) ? 1 : 0;

    #
    # If /usr2 is being restored, then two things have to happen:
    # 1) RTI must be stopped
    # 2) the log file location must be moved
    #
    my $old_logfile_path = "";

    unless ($DRY_RUN) {
	if ($clobber_rti) {

	    # stop RTI
	    if (-f "$RTIDIR/bin/killem") {
		loginfo("killing rti processes: $RTIDIR/bin/killem");
		system("$RTIDIR/bin/killem >> $LOGFILE 2>> $LOGFILE");
	    }

	    # stop RTI services
	    my @rti_services = qw(
		httpd
		bbj
		rti
	    );

	    foreach my $service_name (@rti_services) {
		if (-f "/etc/rc.d/init.d/$service_name") {
		    system("/sbin/service $service_name stop >> $LOGFILE 2>> $LOGFILE");
		    if ($? == 0) {
			loginfo("service stopped: $service_name");
		    }
		    else {
			logerror("could not stop service: $service_name");
		    }
		}
	    }

	    # move logfile location
	    $old_logfile_path = log_change_location();
	}
    }

    # Should we *not* restore some files?
    if (-d "/usr2/basis") {
	push(@{$restore_excludes}, "/usr2/basis");
    }

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ($rc == 0) {
	    if ($clobber_rti) {
		# arrange to run both rti_perms.pl
		$RUN_RTI_PERMS = 1;

		if ($old_logfile_path ne "") {
		    log_restore_location($old_logfile_path);
		}

		showinfo("*** Make sure to restart 'bbj', 'rti' and 'httpd' services.");
	    }

	    #
	    # Make correct symlink for tcc based on which OS we are on.
	    #
	    loginfo("Making TCC links for: $OS");
	    if ($OS eq "RHEL4") {
		if (-e "/usr2/bbx/bin/tcc2_linux") {
		    system("ln -sf /usr2/bbx/bin/tcc2_linux /usr2/bbx/bin/tcc");
		    system("ln -sf /usr2/bbx/bin/tcc_linux /usr2/bbx/bin/tcc_tws");
		}
		else {
		    system("ln -sf /usr2/bbx/bin/tcc_linux /usr2/bbx/bin/tcc");
		}
	    }

	    elsif ($OS eq "RHEL5") {
		if (-e "/usr2/bbx/bin/tcc2_x64") {
		    system("ln -sf /usr2/bbx/bin/tcc2_x64 /usr2/bbx/bin/tcc");
		    system("ln -sf /usr2/bbx/bin/tcc_x64 /usr2/bbx/bin/tcc_tws");
		}
		else {
		    system("ln -sf /usr2/bbx/bin/tcc_x64 /usr2/bbx/bin/tcc");
		}
	    }

	    elsif ($OS eq "RHEL6") {
		system("ln -sf /usr2/bbx/bin/tcc2_rhel6 /usr2/bbx/bin/tcc");
		system("ln -sf /usr2/bbx/bin/tcc_rhel6 /usr2/bbx/bin/tcc_tws");
	    }

	    else {
		logerror("Can't make TCC links: unsupported platform: $OS");
	    }
	}
    }

    loginfo("END Restore RTI /usr2");

    return($rc);
}


sub tfr_restore_bbxd
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    my $verb = (is_restore_in_place()) ? "Restoring" : "Retrieving";
    showinfo("$verb Directory /usr2/bbx/bbxd...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    loginfo("END $verb Directory /usr2/bbx/bbxd");

    return($rc);
}


sub tfr_restore_bbxps
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    my $verb = (is_restore_in_place()) ? "Restoring" : "Retrieving";
    showinfo("$verb Directory /usr2/bbx/bbxps...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    loginfo("END $verb Directory /usr2/bbx/bbxps");

    return($rc);
}


# Restore RTI Config Files
sub tfr_restore_rticonfigs
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restoring RTI Configs...");

    unless ($DRY_RUN) {
	if (is_restore_in_place()) {
	    # must stop RTI
	    if (-f "$RTIDIR/bin/killem") {
		system("$RTIDIR/bin/killem 2>> $LOGFILE");
	    }
	    if (-f "/etc/rc.d/init.d/rti") {
		system("/sbin/service rti stop 2>> $LOGFILE");
	    }
	}
    }

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ( ($rc == 0) && is_restore_in_place() ) {
	    system("/sbin/chkconfig --add --level 35 rti 2>> $LOGFILE");
	    system("/sbin/chkconfig --add --level 35 blm 2>> $LOGFILE");
	    system("/sbin/chkconfig --add --level 35 bbj 2>> $LOGFILE");

	    $RUN_HARDEN_LINUX = 1;
	    $RUN_RTI_PERMS = 1;

	    showinfo("*** Make sure to restart 'rti' service");
	}
    }

    loginfo("END Restore RTI Configs");

    return($rc);
}


# Restore Daisy /d
sub tfr_restore_daisy
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restoring Daisy /d...");

    #
    # Only bring down Daisy if restoring on top of daisy...  which
    # is true only if restoring to the top of the file system - the
    # default value for the restore root directory.
    #
    my $clobber_daisy = (is_restore_in_place()) ? 1 : 0;

    my $old_logfile_path = "";

    unless ($DRY_RUN) {
	if ($clobber_daisy) {
	    # move logfile location
	    $old_logfile_path = log_change_location();

	    # stop daisy
	    system("/sbin/init 4");
	    # wait is suggested by init(1) man page
	    system("sleep 10");

	    if (-e "/d/daisy") {
		system("$DAISYDIR/utils/killemall 2>> $LOGFILE");
	    }
	}
    }

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ($rc == 0) {
	    if ($clobber_daisy) {
		# restart Daisy
		system("/sbin/init 3");
		# wait is suggested by init(1) man page
		system("sleep 10");

		# arrange to run both dsy_perms.pl and harden_linux.pl
		$RUN_HARDEN_LINUX = 1;
		$RUN_DAISY_PERMS = 1;

		if ($old_logfile_path ne "") {
		    log_restore_location($old_logfile_path);
		}
	    }
	}
    }

    loginfo("END Restore Daisy Data");

    return($rc);
}


# Restore Daisy Config Files
sub tfr_restore_dsyconfigs
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restoring Daisy Config Files...");

    #
    # Only stop Daisy if restoring on top of /d...  which
    # is true only if restoring in place.
    #
    my $clobber_daisy = (is_restore_in_place()) ? 1 : 0;

    unless ($DRY_RUN) {
	if ($clobber_daisy) {
	    # stop Daisy
	    system("/sbin/init 4");
	    # wait is suggested by init(1) man page
	    system("sleep 10");

	    if (-e "/d/daisy") {
		system("$DAISYDIR/utils/killemall 2>> $LOGFILE");
	    }
	}
    }

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ($rc == 0) {
	    if ($clobber_daisy) {
		# restart Daisy
		system("/sbin/init 3");
		# wait is suggested by init(1) man page
		system("sleep 10");

		# arrange to run both dsy_perms.pl and harden_linux.pl
		$RUN_HARDEN_LINUX = 1;
		$RUN_DAISY_PERMS = 1;
	    }
	}
    }

    loginfo("END Restore Daisy Configs");

    return($rc);
}


# Restore OS User Config Files
sub tfr_restore_userconfigs
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;


    showinfo("Restoring OS User Configs...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove from list of files to restore those files that
    # are almost certainly going to be different when upgrading,
    # especially when upgrading between os releases
    if ($RestoreUpgradeCLO) {
	my @userconfig_excludes = qw(
	    /etc/pam.d
	    /etc/login.defs
	    /etc/shadow
	    /etc/shadow-
	    /etc/gshadow
	    /etc/gshadow-
	    /etc/passwd
	    /etc/passwd-
	    /etc/group
	    /etc/group-
	);

	# inefficient - loop through entire list for each file
	# to be excluded!  thankfully, list is short.
	foreach my $exclude (@userconfig_excludes) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ( ($rc == 0) && is_restore_in_place() ) {
	    if ($OS eq "RHEL6") {
		my @shadow_files = qw( /etc/shadow /etc/shadow- /etc/gshadow /etc/gshadow- );
		if (tfr_file_perms_unset_readable(\@shadow_files)) {
		    loginfo("[restore userconfigs] shadow files restored to unreadable");
		}
		else {
		    logerror("[restore userconfigs] could not restore shadow files to unreadable");
		}
	    }

	    loginfo("restarting crond...");
	    system("/sbin/service crond restart 2>> $LOGFILE");
	}
    }

    loginfo("END Restore OS User Configs");

    return($rc);
}


# Restore User Specified Files
sub tfr_restore_userfiles
{ 
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = 0;

    showinfo("Restoring User Specified Files...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    if (scalar(@torestore)) {
	$rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);
    }
    else {
	loginfo("[restore user files] no user files specified");
    }

    loginfo("END Restore User Files");

    return($rc);
}


# Restore OS Log Files
#
# Returns
#   0 on success
#   non-zero on error
#
sub tfr_restore_logfiles
{ 
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = 0;

    if ($RestoreUpgradeCLO) {
	loginfo("log files NOT restored: --restore-upgrade specified");
    }
    else {
	showinfo("Restoring Log Files...");

	my @torestore = tfr_backup_file_list($restore_type);

	$rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

	loginfo("END Restore Log Files");
    }

    return($rc);
}


# Restore OS Config Files
sub tfr_restore_osconfigs
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;
    my $datestamp = "";

    showinfo("Restoring OS Configs...");

    # What *not* to restore
    if(-f "/etc/httpd/conf.d/rti.conf") {
	push(@{$restore_excludes}, "/etc/httpd/conf.d/rti.conf");
    }
    if(-f "/etc/sysconfig/rhn/systemid") {
	push(@{$restore_excludes}, "/etc/sysconfig/rhn");
    }

    # if restoring on top of OS locations, copy critical files
    # before blowing them away.
    if (is_restore_in_place()) {
	$datestamp = POSIX::strftime("%Y-%m-%d_%H%M%S", localtime());
	system("cp /etc/inittab /etc/inittab.$datestamp");
    }

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    #
    # some messiness: if restoring FROM RHEL5 TO RHEL6,
    # then some files will have to be excluded.
    #
    # if source platform of backup set == RHEL5 and
    # dst platform == RHEL6 then
    #	    remove "rsyslog.conf" from restore list

    if ($RestoreUpgradeCLO) {
	my $restored_pserver_info_dir_path = tfrm_pathto_restored_pserver_info_dir();
	my $pserver_info_file_path = tfrm_pathto_pserver_info_file();
	my $restored_pserver_info_file_path = File::Spec->catdir($restored_pserver_info_dir_path,
								 $pserver_info_file_path);
	my %pserver_info = ();
	loginfo("[restore osconfigs] restored pserver info file path: $restored_pserver_info_file_path");
	if (tfr_read_pserver_info_file($restored_pserver_info_file_path, \%pserver_info)) {
	    loginfo("[restore osconfigs] restored pserver info file read: $restored_pserver_info_file_path");
	    if ( ($pserver_info{$SERVER_INFO_PLATFORM} eq 'RHEL5') && ($OS eq 'RHEL6') ) {
		@torestore = grep { $_ ne '/etc/rsyslog.conf' } @torestore;
	    }
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ( ($rc == 0) && is_restore_in_place() ) {

	    my $service_name = ($OS eq "RHEL6") ? 'rsyslog' : 'syslog';
	    system("/sbin/service $service_name restart 2>> $LOGFILE");
	    system("/sbin/service rhnsd restart 2>> $LOGFILE");
	    system("/sbin/service sendmail restart 2>> $LOGFILE");

	    if ($RestoreUpgradeCLO && ($OS eq "RHEL6")) {
		my $conf_file = '/etc/samba/smb.conf';
		tfr_restore_upgrade_samba_conf($conf_file);
		$conf_file = '/etc/samba/smbpasswd';
		tfr_restore_upgrade_samba_rebuild_passwd($conf_file);
	    }

	    system("/sbin/service smb restart 2>> $LOGFILE");

	    if ($RTI) {
		if(-f "/etc/rc.d/init.d/httpd 2>> $LOGFILE") {
		    system("/sbin/service httpd restart 2>> $LOGFILE");
		}
	    }

	    my $posdir = ($RTI) ? $RTIDIR : $DAISYDIR;

	    # (Re) Setup tfremote
	    if (-f "$posdir/bin/tfremote.pl") {
		system("$posdir/bin/tfremote.pl --install 2>> $LOGFILE");
	    }
	    else {
		logerror("Error: expecting script to exist: $posdir/bin/tfremote.pl");
		logerror("Error: could not install: $posdir/bin/tfremote.pl");
	    }

	    $RUN_HARDEN_LINUX = 1;
	}
    }

    loginfo("END Restore OS Configs");

    return($rc);
}


#
# Don't preserve the "HWADDR=" line in ifcfg-eth0. This causes issues
# when migrating ethernet configs from an older server to a new server.
#
sub tfr_edit_ifcfg
{
    my $oldfile = "/etc/sysconfig/network-scripts/ifcfg-eth0";
    my $newfile = "/etc/sysconfig/network-scripts/ifcfg-eth0.$$";

    if (open(my $old, '<', $oldfile)) {
	if (open(my $new, '>', $newfile)) {
	    while(<$old>) {
		if (/HWADDR/) {
		    print {$new} "# ";
		}
		print $new;
	    }
	    close($new);
	}
	close($old);
    }

    if (-s $newfile > 0) {
	system("mv $newfile $oldfile");
    }

    return(1);
}


# Restore OS Network Config Files
sub tfr_restore_netconfigs
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restoring Network Configs...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ( ($rc == 0) && is_restore_in_place() ) {
	    tfr_edit_ifcfg();

	    $RUN_HARDEN_LINUX = 1;
	}
    }

    loginfo("END Restore Network Configs");

    return($rc);
}


# Restore Print Config Files
sub tfr_restore_printconfigs
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restoring Printer Configs...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    unless ($DRY_RUN) {
	if ( ($rc == 0) && is_restore_in_place() ) {
	    system("/sbin/service cups restart 2>> $LOGFILE");
	}
    }

    loginfo("END Restore Printer Configs");

    return($rc);
}


#
# If user specified:
#
#   --restore=singlefile --singlefile="/usr2/bbx/blah,/etc/sysconfig/blah"
#
# then this subroutine restores those specific files.
#
# Restore a single file from the backup server or the backup device.
#
sub tfr_restore_singlefile
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("Restoring Single Files...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    loginfo("End Restore Single Files");

    return($rc);
}


sub tfr_restore_specialfiles
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("restoring special files...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    loginfo("end restore special files");

    return($rc);
}


sub tfr_restore_pserver_info
{
    my ($restore_type, $restore_excludes, $restore_dir) = @_;

    my $rc = -1;

    showinfo("restoring pserver info...");

    my @torestore = tfr_backup_file_list($restore_type);

    # remove elements in the "files to be excluded" list from
    # the "files to be restored" list.
    if ($RestoreUpgradeCLO) {
	foreach my $exclude (@{$restore_excludes}) {
	    @torestore = grep { $_ ne $exclude } @torestore;
	}
    }

    $rc = tfr_restore_transaction(\@torestore, $restore_excludes, $restore_type, $restore_dir);

    loginfo("end restore pserver info");

    return($rc);
}


#
# Verify if specified device is on USB bus.
#
# Scan the output of command reporting USB bus info and
# look for the pattern:
#   ID_BUS=usb
# to verify it's on the USB bus.
#
# Returns
#   1 if device is on USB bus
#   0 if not
#
sub is_on_usb_bus
{
    my ($dev_file) = @_;

    my $rc = 0;

    # first choose the command to get udev info depending on platform
    my $udev_cmd = '/usr/bin/udevinfo';
    my $udev_opt = "";
    if ($OS eq 'RHEL6') {
	$udev_cmd = '/sbin/udevadm';
	$udev_opt = 'info';
    }

    # then verify it exists
    if (! -f $udev_cmd) {
	logerror("command to get udev info does not exist: $udev_cmd");
	return($rc);
    }

    if (open(my $pipe, '-|', "$udev_cmd $udev_opt -q env -n $dev_file")) {
	while (<$pipe>) {
	    if (/ID_BUS=usb/) {
		$rc = 1;
		last;
	    }
	}
	close($pipe);
    }
    else {
	logerror("can't open pipe to $udev_cmd for device file: $dev_file");
    }

    return($rc);
}


#
# Actually read the disk and return the filesystem label.
#
# Returns
#   filesystem label  on device
#   empty string if no label
#
sub get_fs_label
{
    my ($dev_file) = @_;

    my $rc = "";

    # verify command to read filesystem label exists
    my $e2label_cmd = '/sbin/e2label';
    if (! -f $e2label_cmd) {
	logerror("command to read filesystem label does not exist: $e2label_cmd");
	return($rc);
    }

    if (open(my $pipe, '-|', "$e2label_cmd $dev_file")) {
	$rc = <$pipe>;
	chomp($rc);
	close($pipe)
    }
    else {
	logerror("can't open pipe to $e2label_cmd for device file: $dev_file");
    }

    return($rc);
}


#
# Get the filesystem label of the specified block device and
# verify if it has a Teleflora label.
#
# Look for the pattern:
#   TFBUDSK-YYYYMMDD
# to verify it has a Teleflora file system label.
#
# Returns
#   1 if filesystem on device has Teleflora label
#   0 if not
#
sub is_teleflora_fs_label
{
    my ($dev_file) = @_;

    my $rc = 0;

    my $fs_label = get_fs_label($dev_file);
    if ($VERBOSE) {
	if ($fs_label eq "") {
	    showinfo("there is no filesystem label for device: $dev_file");
	}
	else {
	    showinfo("filesystem label for device $dev_file: $fs_label");
	}
    }
    if ($fs_label =~ /^$TELEFLORA_FS_LABEL_PATTERN/) {
	$rc = 1;
    }

    return($rc);
}


#
# Locate a USB disk with a Teleflora label - the disk could be either
# a disk or flash drive.
#
# Returns:
#   device file name if disk found is on USB bus and has Teleflora label
#   empty string if not
#
sub find_usb_device
{
    my $returnval = "";
    my @dev_files = qw(
	/dev/sda
	/dev/sdb
	/dev/sdc
	/dev/sdd
	/dev/sde
	/dev/sdf
	/dev/sdg
    );

    # now run through the list of potential device files
    foreach my $this_dev_file (@dev_files) {

	# verify there is a block device file
	next unless (-b $this_dev_file);

	# if on USB bus and has Teleflora filesystem label,
	# then we found one.
	if (is_on_usb_bus($this_dev_file)) {
	    if (is_teleflora_fs_label($this_dev_file)) {
		$returnval = $this_dev_file;
		last;
	    }
	}
    }

    return($returnval);
}


#
# Look at values for "vendor" and "model" of block device.
#
# Note it is important to look at the model. Some shops could have internal Western
# digital HDDs. If we were to only look for "WD" drive, and not look for this particular
# model, then, we could inadvertently use the internal disk device as the "backup" device,
# which would be catastrophic.
#
# Returns
#   1 if vendor == "WD" and model == "Passport"
#   0 if not
#   -1 if error, can't continue
#
sub is_wd_passport
{
    my ($dev_node) = @_;

    unless (-d "/sys/block") {
	logerror("Warning: Cannot look for Passport device: \"/sys\" filesystem not present.");
	return(-1);
    }

    # strip "/dev"
    my $dev_name = basename($dev_node);

    # constuct path to vendor file
    my $sys_vendor_file = "/sys/block/$dev_name/device/vendor";

    # if BOTH Vendor...
    # ... AND Model is correct THEN found one.
    my $found = 0;
    if (-f $sys_vendor_file) {
	if (open(my $file, '<', $sys_vendor_file)) {
	    while (<$file>) {
		if (/$DEVICE_VENDOR/i) {
		    $found = 1;
		    last;
		}
	    }
	    close($file);
	}
	else {
	    logerror("error opening /sys vendor file: $sys_vendor_file");
	}
    }
    if ($found == 0) {
	return(0);
    }

    # constuct path to model file
    my $sys_model_file = "/sys/block/$dev_name/device/model";
    $found = 0;
    if (-f $sys_model_file) {
	if (open(my $file, '<', $sys_model_file)) {
	    while (<$file>) {
		if (/$DEVICE_MODEL/i) {
		    $found = 1;
		    last;
		}
	    }
	    close($file);
	}
	else {
	    logerror("error opening /sys model file: $sys_model_file");
	}
    }
    if ($found == 0) {
	return(0);
    }

    return(1);
}


#
# Locate a Western Digital Passport Device
#
sub find_passport
{
    foreach my $thisdev ("sda", "sdb", "sdc", "sdd", "sde") {

	my $rc = is_wd_passport($thisdev);
	if ($rc == 1) {
	    $DeviceType = $DEVTYPE_PASSPORT;
	    return("/dev/$thisdev");
	}
	elsif ($rc == -1) {
	    last;
	}
    }

    return("");
}


sub is_mount_required
{
    my ($bu_device_type) = @_;

    if ($bu_device_type eq $DEVTYPE_PASSPORT ||
	$bu_device_type eq $DEVTYPE_REV ||
	$bu_device_type eq $DEVTYPE_USB ||
	$bu_device_type eq $DEVTYPE_IMAGE) {
	return(1);
    }
    return(0);
}


#
# Is the rsync backup device already mounted?
#
# Returns
#   1 means "yes, the device is mounted."
#   0 means "no, the device is not mounted."
#
sub is_rsync_bu_device_mounted
{
    my $rc = 0;

    # Does the backup device mount point exist?
    unless (-e "$MOUNTPOINT") {
	logerror("Backup Device mount point \"$MOUNTPOINT\" not found.");
	return($rc);
    }

    my $proc_mount_file = "/proc/mounts";
    if (open(my $file, '<', $proc_mount_file)) {
	while (<$file>) {
	    if (/$MOUNTPOINT/) {
		$rc = 1;
		last;
	    }
	    if (/$DEVICE/) {
		$rc = 1;
		last;
	    }
	}
	close($file);
    }
    else {
	logerror("error opening /proc mounts file: $proc_mount_file");
    }

    return($rc);
}


#
# Mount either a block device, or an "image file".
#
# Returns
#   0 means "yes, mount succeeded"
#   non-zero means "no, the mount failed."
#
sub mount_device
{
    my ($mount_opt) = @_;

    # Is our mountpoint present?
    unless (-d $MOUNTPOINT) {
	showinfo("Making mountpoint: $MOUNTPOINT");
	mkdir("$MOUNTPOINT");
    }
    unless (-d $MOUNTPOINT) {
	showerror("Could not find nor make mountpoint: $MOUNTPOINT");
	return(-3);
    }

    if (is_rsync_bu_device_mounted()) {
	showinfo("Device already mounted: $DEVICE");
	system("umount $DEVICE > /dev/null 2>&1");
	if ($? != 0) {
	    showerror("Could not umount device: $DEVICE");
	    return(-3);
	}
    }

    my $device_type = "";
    if (-b $DEVICE) {
	$device_type = "device";
    }
    elsif (-f $DEVICE) {
	$device_type = "image file";
	$mount_opt .= ",loop";
    }
    else {
	showerror("Unknown device: $DEVICE");
	return(-3);
    }

    if ($VERBOSE) {
	showinfo("Mounting \"$DEVICE\" -> \"$MOUNTPOINT\"");
    }

    system("mount -t ext2 -o $mount_opt $DEVICE $MOUNTPOINT > /dev/null 2>&1");
    if ($? != 0) {
	showerror("Could not mount $device_type: $DEVICE on mountpoint: $MOUNTPOINT");
	return(-3);
    }

    return(0);
}


sub unmount_device
{
    my $returnval = -1;

    if ($DEVICE ne "") {
	if (is_rsync_bu_device_mounted()) {
	    if ($VERBOSE) {
		showinfo("Un-mounting \"$DEVICE\"");
	    }

	    system("umount $DEVICE");
	    $returnval = $?;
	}
    }

    return($returnval);
}


#
# Run the harden script on the POS, with an input argument of either
# "$DAISYDIR" or "$RTIDIR".
#
sub run_harden_linux
{
    my $posdir = "";
    my $rc = 1;

    if ($RTI) {
	$posdir = $RTIDIR;
    }
    elsif ($DAISY) {
	$posdir = $DAISYDIR;
    }

    if (-f "$posdir/bin/harden_linux.pl") {
	showinfo("Running $posdir/bin/harden_linux.pl");
	system("perl $posdir/bin/harden_linux.pl 2>> $LOGFILE");
    } else {
	logerror("Error: expecting script to exist: $posdir/bin/harden_linux.pl");
	logerror("Error: could not run: $posdir/bin/harden_linux.pl");
	$rc = 0;
    }

    return($rc);
}


#
# Run rtiperms.pl on the RTI application files.
#
sub set_rti_perms
{
    my $rc = 1;

    if (-f "$RTIDIR/bin/rtiperms.pl") {
	showinfo("Running $RTIDIR/bin/rtiperms.pl /usr2/bbx");
	system("perl $RTIDIR/bin/rtiperms.pl /usr2/bbx 2>> $LOGFILE");
    } else {
	logerror("Error: expecting script to exist: $RTIDIR/bin/rtiperms.pl");
	logerror("Error: could not run: $RTIDIR/bin/rtiperms.pl /usr2/bbx");
	$rc = 0;
    }

    return($rc);
}


#
# Run dsyperms.pl on all possible daisy database dirs... always remember
# that there can be multiple daisy database dirs!
#
# Look for all directories in "/d" that contain files named
# "flordat.tel" and "control.dsy".  Skip old dirs.
#
sub set_daisy_perms
{
	my @daisy_db_dirs = glob("$DAISY_TOPDIR/*");
	my $dsyperms_cmd = "$DAISY_BINDIR/dsyperms.pl";
	my $rc = 1;

	for my $daisy_db_dir (@daisy_db_dirs) {

		# must be a directory
		next unless (-d $daisy_db_dir);

		# skip old daisy dirs
		next if ($daisy_db_dir =~ /.+-\d{12}$/);

		# must contain the magic files
		next unless(-e "$daisy_db_dir/flordat.tel");
		next unless(-e "$daisy_db_dir/control.dsy");

		if (-f "$DAISY_BINDIR/dsyperms.pl") {
			showinfo("Running $dsyperms_cmd $daisy_db_dir");
			system("perl $dsyperms_cmd $daisy_db_dir 2>> $LOGFILE");
		} else {
			logerror("Error: expecting script to exist: $dsyperms_cmd");
			logerror("Error: could not run: $dsyperms_cmd");
			$rc = 0;
		}
	}

    return($rc);
}

#
# Function to determine if an arbitrary path is a path to a
# daisy databse directory.
#
# MOVE TO MODULE
#
sub is_daisy_db_dir
{
	my ($path) = @_;

	# must begin with '/d/'
	return(0) unless ($path =~ /^\/d\//);

	# must be a directory
	return(0) unless (-d $path);

	# skip old daisy dirs
	return(0) if ($path =~ /^\/d\/.+-\d{12}$/);

	# must contain the magic files
	return(0) unless (-e "$path/flordat.tel");
	return(0) unless (-e "$path/control.dsy");

	# must be daisy 8.0+
	return(0) unless (-d "$path/bin");

	return(1);
}


sub tfr_send_email
{
    my ($subject, $message) = @_;

    my $rc = 1;

    return($rc) if ($DRY_RUN);

    return($rc) unless (is_configured_email_recipients());

    # tack the system hostname onto end of the subject.
    my $hostname = hostname();
    $subject .= " (host $hostname)";

    # first choice, send via "sendmail"
    if (is_configured_sendmail()) {
	# Use the sendmail program directly
	if (send_email_sendmail($subject, $message)) {
	    loginfo("email sent via sendmail: $subject");
	}
	else {
	    logerror("could not send email via sendmail: $subject}");
	    $rc = 0;
	}
    }

    # second choice via configured SMTP server with credentials
    # If configured to do so, try sending email via smtp server
    elsif (is_configured_smtp_mail()) {
	if (send_email_smtp($subject, $message)) {
	    loginfo("email sent via SMTP server: $subject}");
	}
	else {
	    logerror("could not send email via SMTP server: $subject}");
	    $rc = 0;
	}
    }
    else {
	logerror("could not send email message - email not configured");
	$rc = 0;
    }

    return($rc);
}


#
# Send email via direct connect from this script, to an SMTP(s) server.
# This "way" of sending email totally bypasses the use of sendmail, and
# could also resolve issues with email blacklisting.
#
# Returns
#   1 on success
#   0 on error
#
sub send_email_smtp
{
    my ($subject, $message) = @_;

    my $hostname = hostname();
    my $rc = 1;

    foreach my $recipient (@EMAIL_RECIPIENTS) {

	# Connect to an SMTP server.
	my $smtp = Net::SMTP->new($EMAIL_SERVER, Port=> 25);
	if ($smtp) {
	    $smtp->auth($EMAIL_USER, $EMAIL_PASS);
	    $smtp->mail("backups\@$hostname");
	    $smtp->to("$recipient\n", {SkipBad => 1} );
	    $smtp->data();
	    $smtp->datasend("From: backups\@$hostname\n");
	    $smtp->datasend("To: $recipient\n");
	    $smtp->datasend("Subject: $subject\n");
	    $smtp->datasend("\n");
	    $smtp->datasend("$message\n");
	    $smtp->dataend();
	    $smtp->quit;
	    loginfo("email sent via SMTP server to: $recipient");
	}
	else {
	    logerror("could not connect to SMTP server: $EMAIL_SERVER");
	    $rc = 0;
	}
    }

    return($rc);
}


#
# Send email via pipe to "sendmail" command.
#
# Returns
#   1 on success
#   0 on error
#
sub send_email_sendmail
{
    my ($subject, $message) = @_;

    my $rc = 1;

    my $sendmail_cmd = '/usr/lib/sendmail';
    unless (-x $sendmail_cmd) {
	logerror("could not send email: sendmail command does not exist: $sendmail_cmd");
	return(0);
    }

    my $hostname = hostname();
    my $from = "$PROGNAME\@${hostname}.teleflora.com";

    my $cmd = "$sendmail_cmd -oi -t";

    foreach my $recipient (@EMAIL_RECIPIENTS) {
	if (open(my $mail, '|-', $cmd)) {
	    print {$mail} "From: $from\n";
	    print {$mail} "To: $recipient\n";
	    print {$mail} "Subject: $subject\n\n";
	    print {$mail} "$message\n";
	    close($mail);
	    loginfo("email sent via sendmail to: $recipient");
	}
	else {
	    logerror("could not connect to sendmail cmd: $cmd");
	    $rc = 0;
	}
    }

    return($rc);
}


#
# Send backup results to a list of printers.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_print_results
{
    my ($subject, $message) = @_;

    my $rc = 1;

    unless (is_configured_print()) {
	return($rc);
    }

    my $timestamp = POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime());

    my @template = (
	"=============================================================\r\n",
	"$subject\r\n",
	"Begin Printout:",
	"\r\n",
	"=============================================================\r\n",
	"\r\n",
	"\r\n",
	"\r\n",
	"\r\n",
	"\r\n",
	"$message",
	"\r\n",
	"\r\n",
	"\r\n",
	"\r\n",
	"\r\n",
	"=============================================================\r\n",
	"End Printout: $timestamp\r\n",
	"$subject\r\n",
	"=============================================================\r\n",
    );

    foreach my $printer (@PRINTERS) {

	next if ($printer eq "");

	my $print_cmd = "lp -d $printer";
	if (open(my $pipe, '|-', $print_cmd)) {
	    foreach (@template) {
		print {$pipe} "$_";
		if (/^Begin Printout:/) {
                    print {$pipe} " $timestamp on printer: $printer";
		}
	    }
	    close($pipe);
	}
	else {
	    $rc = 0;
	}
    }

    return($rc);
}


#
# make the dir at the given path if it does not
# already exist.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_util_mkdir
{
    my ($dir_path) = @_;

    my $rc = 1;

    unless (-d $dir_path) {
	mkdir($dir_path);
	unless (-d $dir_path) {
	    $rc = 0;
	}
    }

    return($rc);
}


#
# remove the pserver dir at the given path
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_util_rmdir
{
    my ($dir_path) = @_;

    my $rc = 1;

    if (-d $dir_path) {
	system("rm -rf $dir_path");
	if (-d $dir_path) {
	    $rc = 0;
	}
    }

    return($rc);
}


#
# form the cloud account name.  The cloud account name is:
#
#   project name . '-' . shopcode
#
# EG, tfrsync-01234567
#
# Returns
#   account name on success
#   empty string on failure
#
sub tfr_cloud_account_name
{
    my $cloud_account_name = "";

    my $shopcode = get_shopcode();
    if ($shopcode) {
	$cloud_account_name = $DEF_PROJ_NAME . '-' . $shopcode;
    }

    return($cloud_account_name);
}


#
# Given an account name, verify existence of account.
#
# Returns
#   1 if account exists
#   0 if account does NOT exist
#
sub tfr_accounts_exists
{
    my ($account_name) = @_;

    my $system_uid = getpwnam($account_name);
    return((defined($system_uid)) ? 1 : 0);
}


#
# Given an account name, return path to home directory.
#
# Returns
#   path to home dir if account exists
#   empty string if account does NOT exist
#
sub tfr_accounts_homedir
{
    my ($account_name) = @_;

    my $homedir_path = (getpwnam($account_name))[7];
    if (!defined($homedir_path)) {
	$homedir_path = "";
    }

    return($homedir_path);
}


#
# Given an account name, get the path to default ssh dir.
#
# Returns
#   path to default ssh dir on success
#   empty string on failure
#
sub tfr_sshdir_default_path
{
    my ($account_name) = @_;

    my $sshdir_path = "";

    if ($account_name) {
	if (tfr_accounts_verify_account($account_name)) {
	    my $homedir_path = tfr_accounts_homedir($account_name);
	    $sshdir_path = $homedir_path . '/' . ".ssh";
	}
    }

    return($sshdir_path);
}


#
# Given an account name, make the default ssh dir if it does not
# already exist.
#
# Returns
#   path to ssh dir on success
#   empty string on error
#
sub tfr_accounts_make_default_sshdir
{
    my ($account_name) = @_;

    my $account_sshdir_path = tfr_sshdir_default_path($account_name);

    if (-d $account_sshdir_path) {
	loginfo("default ssh dir for account already exists: $account_name");
    }
    else {
	# make the dir and then verify that the dir exists
	system("mkdir $account_sshdir_path");
	if (-d $account_sshdir_path) {
	    # mkdir successful, set owners and perms
	    loginfo("mkdir of default ssh dir successful for account: $account_name");
	    my $account_homedir = dirname($account_sshdir_path);
	    system("chown --reference $account_homedir $account_sshdir_path");
	    system("chmod 0700 $account_sshdir_path");
	}
	else {
	    showerror("mkdir of default ssh dir failed for account; $account_name");
	    $account_sshdir_path = "";
	}
    }

    return($account_sshdir_path);
}


#
# Given an account name, verify it's existence.
#
# Returns
#   1 if account exists
#   0 if account does not exist
#   
sub tfr_accounts_verify_account
{
    my ($account_name) = @_;

    if (system("id -u $account_name > /dev/null 2>&1") == 0) {
	return(1);
    }

    return(0);
}


#
# Given an account name, add a system account with that name.
#
# BE AWARE: when adding an account, the "useradd" command can be
# netatively influenced by the contents of "/etc/login.defs" and
# "/etc/default/useradd".  For example, if "/etc/login.defs" has
# a low value for "PASS_MAX_DAYS", then this script will not work
# after the password expires even tho password-less ssh keys are
# being used for authentication.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_accounts_add_account
{
    my ($account_name, $account_full_name) = @_;

    my $rc = 1;

    # if the account does not exist, try to add it, and verify
    if (tfr_accounts_verify_account($account_name) == 0) {
	my $cmd = "/usr/sbin/useradd";
	my $cmd_opts = "-c \"$account_full_name\"";
	system("$cmd $cmd_opts $account_name");
	if (tfr_accounts_verify_account($account_name) == 0) {
	    logerror("command failed: $cmd $cmd_opts $account_name");
	    $rc = 0;
	}
    }

    return($rc);
}


#
# Given an account name, remove it from the system.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_accounts_rm_account
{
    my ($rsync_account_name) = @_;
    my $rc = 1;

    # if the account exists, remove it
    if (tfr_accounts_verify_account($rsync_account_name)) {
	my $cmd = "/usr/sbin/userdel";
	my $cmd_opts = "-r";
	system("$cmd $cmd_opts $rsync_account_name");
	if (tfr_accounts_verify_account($rsync_account_name)) {
	    showerror("error removing account: $rsync_account_name");
	    $rc = 0;
	}
    }
    else {
	loginfo("account does not exist: $rsync_account_name");
    }

    return($rc);
}


#####################################
####### SERVER INFO FILE SUBS #######
#####################################

#
# toplevel function for getting the contents of a
# server info file and writing it to the specified path.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_generate_pserver_info_file
{
    my ($pserver_info_file_path) = @_;

    my $rc = 0;

    my %server_info = ();
    if (tfr_prepare_pserver_info_file(\%server_info)) {
	if (tfr_write_pserver_info_file($pserver_info_file_path, \%server_info)) {
	    loginfo("[write pserver info file] server info file written: $pserver_info_file_path");

	    $rc = 1;
	}
	else {
	    showerror("[write pserver info file] could not write server info file: $pserver_info_file_path");
	}
    }
    else {
	showerror("[write pserver info file] could not prepare pserver info file: $pserver_info_file_path");
    }

    return($rc);
} 


#
# prepare info for the production server info file by
# putting it into the specified hash ref.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_prepare_pserver_info_file
{
    my ($server_info_ref) = @_;

    my $rc = 0;

    my $ip_addr = get_ip_address();
    if ($ip_addr) {
	loginfo("ip addr of primary network device: $ip_addr");
	my $netmask = get_netmask();
	if ($netmask) {
	    loginfo("netmask of primary network device: $netmask");
	    my $gateway_ip_addr = get_gateway_ipaddr();
	    if ($gateway_ip_addr) {
		loginfo("ip addr of gateway: $gateway_ip_addr");
		my $hostname = hostname();

		$server_info_ref->{$SERVER_INFO_PLATFORM} = $OS;
		$server_info_ref->{$SERVER_INFO_HOSTNAME} = $hostname;
		$server_info_ref->{$SERVER_INFO_IPADDR} = $ip_addr;
		$server_info_ref->{$SERVER_INFO_NETMASK} = $netmask;
		$server_info_ref->{$SERVER_INFO_GATEWAY} = $gateway_ip_addr;

		$rc = 1;
	    }
	    else {
		showerror("error getting ip addr of gateway");
	    }
	}
	else {
	    logerror("error getting netmask for primary network device");
	}
    }
    else {
	logerror("error getting ip addr for primary network device");
    }

    return($rc);
}


#
# write server info to the file at the given path from
# the given hash reference.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_write_pserver_info_file
{
    my ($pserver_info_file_path, $pserver_info_ref) = @_;

    my $rc = 0;

    if (open(my $fh, '>', $pserver_info_file_path)) {

	print {$fh} "$SERVER_INFO_PLATFORM=$pserver_info_ref->{$SERVER_INFO_PLATFORM}\n";
	print {$fh} "$SERVER_INFO_HOSTNAME=$pserver_info_ref->{$SERVER_INFO_HOSTNAME}\n";
	print {$fh} "$SERVER_INFO_IPADDR=$pserver_info_ref->{$SERVER_INFO_IPADDR}\n";
	print {$fh} "$SERVER_INFO_NETMASK=$pserver_info_ref->{$SERVER_INFO_NETMASK}\n";
	print {$fh} "$SERVER_INFO_GATEWAY=$pserver_info_ref->{$SERVER_INFO_GATEWAY}\n";

	close($fh);

	loginfo("[write pserver info file] pserver info file generated: $pserver_info_file_path");

	$rc = 1;
    }
    else {
	showerror("[write pserver info file] could not open pserver info file: $pserver_info_file_path");
    }

    return($rc);
}


#
# read pserver info file at the given path and
# fill given hash reference.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_read_pserver_info_file
{
    my ($pserver_info_file_path, $pserver_info_ref) = @_;

    my $rc = 0;

    unless (-e $pserver_info_file_path) {
	logerror("[read pserver info file] file does not exist: $pserver_info_file_path");
	return($rc);
    }

    if (open(my $fh, '<', $pserver_info_file_path)) {
	while (<$fh>) {
	    if (/^$SERVER_INFO_PLATFORM=(.+)$/) {
		$pserver_info_ref->{$SERVER_INFO_PLATFORM} = $1;
	    }
	    if (/^$SERVER_INFO_HOSTNAME=(.+)$/) {
		$pserver_info_ref->{$SERVER_INFO_HOSTNAME} = $1;
	    }
	    elsif (/^$SERVER_INFO_IPADDR=(.+)$/) {
		$pserver_info_ref->{$SERVER_INFO_IPADDR} = $1;
	    }
	    elsif (/^$SERVER_INFO_NETMASK=(.+)$/) {
		$pserver_info_ref->{$SERVER_INFO_NETMASK} = $1;
	    }
	    elsif (/^$SERVER_INFO_GATEWAY=(.+)$/) {
		$pserver_info_ref->{$SERVER_INFO_GATEWAY} = $1;
	    }
	}
	close($fh) || logerror("[read pserver info file] close error: $pserver_info_file_path");
	$rc = 1;
    }
    else {
	logerror("[read pserver info file] could not open pserver info file: $pserver_info_file_path");
    }

    return($rc);
}


#
# Given an account name, look for given public key in the
# account's authorized keys file in the default ssh dir.
# Default ssh dir and authorized keys file should exist but
# no error message if they don't.
#
# Returns
#   1 if key found
#   0 if key not found
#
sub tfr_keys_find_key_in_auth_keys
{
    my ($account_name, $key_path) = @_;

    my $auth_keys_path = tfr_pathto_ssh_auth_keys_file($account_name);
    if ($auth_keys_path) {

	# read the public key file
	if (open(my $kf, '<', $key_path)) {
	    my $public_key = <$kf>;
	    close($kf);
	    chomp($public_key);

	    # search for public key in authorized keys file
	    if (open(my $akf, '<', $auth_keys_path)) {
		while (my $line = <$akf>) {
		    chomp($line);
		    if ($public_key eq $line) {
			close($akf);
			return(1);
		    }
                }
		# if get here, did not find it
		close($akf);
            }
	}
    }

    return(0);
}


#
# Given an account name, add the given public key to the
# account's authorized keys file in the default ssh dir.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_keys_add_key_to_auth_keys
{
    my ($account_name, $key_path) = @_;

    my $rc = 1;

    # get the path to default ssh dir, make it if necessary
    my $sshdir_path = tfr_accounts_make_default_sshdir($account_name);
    if ($sshdir_path) {
	if (-e "$key_path") {
	    system("cd $sshdir_path; cat $key_path >> $SSH_AUTH_KEYS_FILENAME");
	    system("cd $sshdir_path; chmod 600 $SSH_AUTH_KEYS_FILENAME");
	    system("cd $sshdir_path; chown $account_name $SSH_AUTH_KEYS_FILENAME");
	    system("cd $sshdir_path; chgrp $account_name $SSH_AUTH_KEYS_FILENAME");
	    if (tfr_keys_find_key_in_auth_keys($account_name, $key_path)) {
		loginfo("key added to authorized keys file for account: $account_name");
	    }
	    else {
		showerror("error adding key to authorized keys file for account: $account_name");
		showerror("path to key: $key_path");
		$rc = 0;
	    }
	}
	else {
	    showerror("public key does not exist: $key_path");
	    $rc = 0;
	}
    }
    else {
	showerror("could not get path to default ssh dir for account: $account_name");
	$rc = 0;
    }

    return($rc);
}


#
# Given the path to an authorized keys file, remove
# the given public key from said authorized keys file.
#
# Returns
#   1 on success
#   0 if error
#
sub tfr_auth_keys_file_rm_key
{
    my ($auth_keys_path, $public_key_path) = @_;

    my $rc = 0;

    my $public_key = $EMPTY_STR;
    my $new_auth_keys_path = $auth_keys_path . '.' . "$$";

    # read the public key file
    if (open(my $kf, '<', $public_key_path)) {
	$public_key = <$kf>;
	chomp($public_key);
	close($kf);
    }
    else {
	showerror("could not open public key file for reading: $auth_keys_path");
    }

    if ($public_key eq $EMPTY_STR) {
	showerror("public key file empty: $public_key_path");
    }
    else {
	# read the authorized keys file
	my @auth_keys = ();
	if (open(my $akf, '<', $auth_keys_path)) {
	    chomp(@auth_keys = <$akf>);
	    close($akf);
	}
	else {
	    showerror("could not open authorized keys file for reading: $auth_keys_path");
	}

	if (scalar(@auth_keys) == 0) {
	    showerror("authorized keys file empty: $auth_keys_path");
	}
	else {
	    # write the authorized keys without public key to new file
	    if (open(my $nakf, '>', $new_auth_keys_path)) {
		for my $auth_key (@auth_keys) {
		    if ($auth_key ne $public_key) {
			print {$nakf} "$auth_key\n";
		    }
		}
		close($nakf);
	    }
	    else {
		showerror("could not open new authorized keys file for writing: $new_auth_keys_path");
	    }
	}
    }

    # if successful, switch files
    if (-e $new_auth_keys_path) {
	if (-s $new_auth_keys_path) {
	    system("chmod --reference=$auth_keys_path $new_auth_keys_path");
	    system("chown --reference=$auth_keys_path $new_auth_keys_path");
	    system("mv $new_auth_keys_path $auth_keys_path");
	    loginfo("public key removed from: $auth_keys_path");
	    $rc = 1;
	}
	else {
	    showerror("new authorized keys file zero length: $new_auth_keys_path");
	    unlink($new_auth_keys_path);
	}
    }

    return($rc);
}


#
# Given an account name, generate a SSH RSA key pair.
# If the default SSH directory does not exist, make it.
# If there are old SSH keys, rename them with a timestamp.
# Generate passwordless keys, of type and bits specified by
# defines at top of program (currently RSA and 2048), and
# set the perms as appropriate.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_keys_generate_ssh_keys
{
    my ($account_name) = @_;

    # get the path to default ssh dir, make it if necessary
    my $sshdir_path = tfr_accounts_make_default_sshdir($account_name);
    if ($sshdir_path eq "") {
	showerror("could not make default ssh dir for account: $account_name");
	return(0);
    }

    # form the path to the private and public keys
    my $keyfile_path = File::Spec->catdir($sshdir_path, $SSH_KEY_FILENAME);
    my $keyfile_path_pub = File::Spec->catdir($sshdir_path, $SSH_KEY_FILENAME_PUBLIC);

    # if old keys exist, rename them with timestamp
    if (-e $keyfile_path) {
	my $st = stat($keyfile_path);
	my $timestamp = POSIX::strftime("%Y%m%d%H%M%S", localtime($st->mtime));
	my $keyfile_prev = $keyfile_path . "-$timestamp";
	my $keyfile_prev_pub = $keyfile_path_pub . "-$timestamp";
	system("rm -f $keyfile_prev $keyfile_prev_pub");
	system("mv $keyfile_path $keyfile_prev");
	system("mv $keyfile_path_pub $keyfile_prev_pub");
    }
    if (-e $keyfile_path) {
	showerror("could not rename old key file for account: $account_name");
	return(0);
    }

    # generate the key pair
    my $comment = $account_name . '@' . hostname;
    my $cmd = "/usr/bin/ssh-keygen";
    my $cmd_opts  = "-b $SSH_KEY_LEN";  # key length
    $cmd_opts .= " -t $SSH_KEY_TYPE";   # key type
    $cmd_opts .= " -N \"\"";            # no passphrase
    $cmd_opts .= " -f $keyfile_path";   # location of key file
    $cmd_opts .= " -C $comment";        # add comment
    loginfo("generate keys cmd: $cmd $cmd_opts");
    system("$cmd $cmd_opts >> $LOGFILE 2>> $LOGFILE");
    if (-e $keyfile_path) {
	loginfo("keys generated: $keyfile_path");
	my $homedir_path = (getpwnam $account_name)[7];
	system("chown --reference $homedir_path $keyfile_path");
	system("chmod 0600 $keyfile_path");
	system("chown --reference $homedir_path $keyfile_path_pub");
	system("chmod 0644 $keyfile_path_pub");
    }
    else {
	showerror("could not generate new keys for account: $account_name");
	return(0);
    }

    return(1);
}


#
# Given an account name and a destination path, copy the
# public key file located in the account's default ssh dir
# to the destination path.
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_keys_copy_public_key_file
{
    my ($account_name, $dst_path) = @_;

    my $rc = 0;

    my $key_path = tfr_pathto_ssh_public_key_file($account_name);
    if ($key_path) {
	# copy the key file
	system("cp $key_path $dst_path");
	if (-e $dst_path) {
	    loginfo("public key file copied: $dst_path");
	    $rc = 1;
	}
	else {
	    showerror("could not copy public key file ($key_path) to: $dst_path");
	}
    }
    else {
	showerror("public key file does not exist for account: $account_name");
    }

    return($rc);
}


#
# Given an account name, remove the private key file, ie,
# the ssh identity file.
#
# Returns
#   1 on success
#   0 on error 
#
sub tfr_keys_rm_private_key
{
    my ($account_name) = @_;

    my $rc = 1;

    my $private_key_path = tfr_pathto_ssh_id_file($account_name);
    if (-e $private_key_path) {

	unlink($private_key_path);
	if (-e $private_key_path) {
	    showerror("can't remove private key: $private_key_path");
	    $rc = 0;
	}
	else {
	    loginfo("private key removed: $private_key_path");
	}
    }
    else {
	loginfo("private key file does not exist for account: $account_name");
    }

    return($rc);
}


#
# Given an account name, remove the public key file.
#
# Returns
#   1 on success
#   0 on error 
#
sub tfr_keys_rm_public_key
{
    my ($account_name) = @_;

    my $rc = 1;

    my $public_key_path = tfr_pathto_ssh_public_key_file($account_name);
    if (-e $public_key_path) {

	unlink($public_key_path);
	if (-e $public_key_path) {
	    showerror("can't remove public key for account: $account_name");
	    $rc = 0;
	}
	else {
	    loginfo("ssh public key removed: $public_key_path");
	}
    }
    else {
	loginfo("public key file does not exist for account: $account_name");
    }

    return($rc);
}


#
# Given an account name, get the public key for the corresponding account
# on the primary server and put it in the authorized keys file in the
# given accounts default ssh dir.
#
# This sub is called during an installation of the backup server so the
# name of the account will be "tfrsync".
#
# When the private/public key pair for the "tfrsync" account was generated
# on the primary server, the public key was put into the home dir of the
# support account in a special directory so it could be copied from the
# primary server to the backup server knowing only the support accounts
# password.
#
# Returns
#   1 on success
#   0 on error 
#
sub tfr_keys_get_primary_public_key
{
    my ($account_name, $primary_server) = @_;

    my $rc = 0;

    # form command to get the public key from the primary server -
    # the primary's "tfrsync" public key was put into the home dir
    # of the support account
    my $cmd = "/usr/bin/scp";
    my $tfsupport = tfr_tfsupport_account_name();
    my $src_host = $tfsupport . '@' . $primary_server . ':';
    my $src_arg = $src_host . $PSERVER_XFER_DIR . '/' . $SSH_KEY_FILENAME_PUBLIC;

    # make the xfer dir for specified account so there
    # is a destination to put the public key file
    my $dst_dir = tfrm_pathto_pserver_xfer_dir();
    if ($dst_dir) {

	# copy the public key file from primary server to backup server
	loginfo("cmd for getting primary public key: $cmd $src_arg $dst_dir");
	system("$cmd $src_arg $dst_dir");

	# was the copy successful?
	my $dst_path = "$dst_dir/$SSH_KEY_FILENAME_PUBLIC";
	if (-e $dst_path) {
	    loginfo("primary public key copied to xfer dir: $dst_path");

	    # put the primary server public key in authorized keys file
	    if (tfr_keys_add_key_to_auth_keys($account_name, $dst_path)) {
		loginfo("primary public key added to authorized keys file for account: $account_name");
		$rc = 1;
	    }
	    else {
		showerror("error adding primary public key to authorized keys file for: $account_name");
	    }
	}
	else {
	    showerror("error copying primary ssh public key to: $dst_path");
	}
    }
    else {
	showerror("error making xfer dir for public key: $dst_dir");
    }

    return($rc);
}


#
# Call the ostools script "harden_linux.pl" to generate a
# new /etc/sudoers file.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_install_sudoers_generate
{
    my $cmd = "$ToolsDir/bin/harden_linux.pl";
    my $cmd_opts = "--sudo";

    loginfo("generating a new sudoers file: $cmd $cmd_opts");
    if (system("$cmd $cmd_opts >> $LOGFILE 2>> $LOGFILE") != 0) {
	logerror("harden_linux.pl returned non-zero exit status: $?");
	return(0);
    }

    return(1);
}


#
# Call the ostools script "harden_linux.pl" to convert the
# "append sudoers" directvies in the "harden_linux.pl"
# config file to use content files in the "sudoers.d"
# directory in the ostools "config" directory.  Why?
# This script must add content to the "/etc/sudoers" file
# and it's much easier to do that with content files rather
# then editing "append" directives in the "harden_linux.pl"
# config file.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_install_sudoers_convert
{
    my $cmd = "$ToolsDir/bin/harden_linux.pl";
    my $cmd_opts = "--convert-configfile";

    loginfo("converting harden_linux.pl config file: $cmd $cmd_opts");
    if (system("$cmd $cmd_opts >> $LOGFILE 2>> $LOGFILE") != 0) {
	logerror("harden_linux.pl returned non-zero exit status: $?");
	return(0);
    }

    return(1);
}


#
# Add the tfrsync account to the "/etc/sudoers" file so that the
# tfrsync account can run the "rsync" command as root.
#
# Steps to add new entry to "sudoers" file via harden_linux.pl:
# 0) verify existence of harden_linux.pl sudoers config dir 
# 1) remove any old sudoers content files in the harden_linux sudoers config dir
# 2) generate new harden_linux.pl sudoers content file
# 3) generate new /etc/sudoers file by running the "harden_linux.pl --sudo" script
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_install_sudoers_add
{
    my (@sudoers_conf_lines) = @_;

    # verify existence of harden_linux.pl sudoers config dir
    my $sudoers_conf_dir = "$ToolsDir/config/sudoers.d";
    my $sudoers_conf_file = "tfrsync.conf";
    if (! -d $sudoers_conf_dir) {
	showerror("harden_linux.pl sudoers directory does not exist: $sudoers_conf_dir");
	return(0);
    }

    # remove the old harden_linux.pl sudoers content file
    my $sudoers_content_path = "$sudoers_conf_dir/$sudoers_conf_file";
    if (-e $sudoers_content_path) {
	if (unlink($sudoers_content_path)) {
	    loginfo("existing tfrsync sudoers content file removed: $sudoers_content_path");
	}
	else {
	    showerror("can't remove existing tfrsync sudoers content file: $sudoers_content_path");
	    return(0);
	}
    }

    # generate new harden_linux.pl sudoers content file
    if (open(my $cf, '>', $sudoers_content_path)) {
	foreach (@sudoers_conf_lines) {
	    print {$cf} "$_\n";
	}
	close($cf);
    }
    else {
	showerror("can't generate new tfrsync sudoers content file: $sudoers_content_path");
	return(0);
    }

    # generate new /etc/sudoers file
    if (tfr_install_sudoers_generate() == 0) {
	showerror("can't generate new /etc/sudoers file");
	return(0);
    }

    return(1);
}


#
# Remove the tfrsync "sudoers" content file in the "harden_linux.pl"
# configuration directory and generate a new sudoers file.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_install_sudoers_rm
{
    my $rc = 1;

    if (-d $SUDOERS_CONFIG_DIR_PATH) {
	my $sudoers_content_path = $SUDOERS_CONTENT_FILE_PATH;
	if (-e $sudoers_content_path) {
	    unlink($sudoers_content_path);
	    if (-e $sudoers_content_path) {
		showerror("error removing tfrsync sudoers content file: $sudoers_content_path");
		$rc = 0;
	    }
	    else {
		loginfo("tfrsync sudoers content file removed: $sudoers_content_path");
	    }
	}
    }

    if (tfr_install_sudoers_generate() == 0) {
	showerror("can't generate new sudoers file");
	$rc = 0;
    }

    return($rc);
}


#
# Configure the "tfrsync" account to be able to run "rsync" via "sudo".
# Steps to accomplish:
# 1) convert harden_linux.pl to use sudoers content files
# 2) add new /etc/sudoers content
#
# Returns
#   1 on success
#   0 on error 
#
sub tfr_install_sudoers_config
{
    my ($rsync_account_name) = @_;

    if (tfr_install_sudoers_convert()) {
	loginfo("the harden_linux.pl config file converted");

	my @sudoers_conf_lines = (
	    "tfrsync     ALL=       NOPASSWD: /usr/bin/rsync",
	    "Defaults:tfrsync    !requiretty",
	);
	if (tfr_install_sudoers_add(@sudoers_conf_lines)) {
	    loginfo("content for account added to sudoers file: $rsync_account_name");
	}
	else {
	    showerror("can't add content for account to sudoers file: $rsync_account_name");
	    return(0);
	}
    }
    else {
	showerror("conversion of harden_linux.pl config file failed");
	return(0);
    }

    return(1);
}


sub tfr_cron_job_type
{
    my ($device_type) = @_;

    my $cron_job_type = "";

    if ($device_type eq $DEVTYPE_SERVER) {
	$cron_job_type = $CRON_JOB_TYPE_SERVER;
    }
    elsif ($device_type eq $DEVTYPE_CLOUD) {
	$cron_job_type = $CRON_JOB_TYPE_CLOUD; 
    }
    else {
	$cron_job_type = $CRON_JOB_TYPE_DEVICE;
    }

    return($cron_job_type);
}


sub tfr_cron_job_path
{
    my ($cron_job_type) = @_;

    my $cron_job_path = "";

    if ($cron_job_type eq $CRON_JOB_TYPE_SERVER) {
	$cron_job_path = $CRON_JOB_SERVER_PATH;
    }
    elsif ($cron_job_type eq $CRON_JOB_TYPE_CLOUD) {
	$cron_job_path = $CRON_JOB_CLOUD_PATH;
    }
    else {
	$cron_job_path = $CRON_JOB_DEVICE_PATH;
    }

    return($cron_job_path);
}


#
# Add a cron job.
#
#   run tfrsync.pl directly from ostools bin directory
#   default frequency: 23:30
#   command line: tfrsync.pl --backup=all
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_cron_job_add
{
    my ($cron_job_type) = @_;

    # If a cron job file already exists, don't overwrite it since
    # it might have site dependent contents.  Put the new version
    # into the ostools config directory.
    my $cron_job_path = tfr_cron_job_path($cron_job_type);
    if ($cron_job_path eq "") {
	showerror("can't determine cron job path for type: $cron_job_type");
	return(0);
    }

    if (-e $cron_job_path) {
	showerror("cron job file already exists: $cron_job_path");
	my $cron_job_file_name = basename($cron_job_path);
	$cron_job_path = $TOOLS_CONFIG_DIR_PATH . '/' . $cron_job_file_name . '-new';
	showinfo("cron job file will be written to: $cron_job_path");
    }

    # form the command for the cron job and it's options
    my $cmd = "$ToolsDir/bin/$PROGNAME";
    my $cmd_opts = "";
    if ($cron_job_type eq $CRON_JOB_TYPE_SERVER) {
	$cmd_opts = "--backup=all --server";
    }
    elsif ($cron_job_type eq $CRON_JOB_TYPE_CLOUD) {
	$cmd_opts = " --backup=all --cloud";
    }
    else {
	$cmd_opts = " --backup=all";
    }

    # Make a new cron job file
    if (open(my $file, '>', $cron_job_path)) {

	print {$file} "#\n";
	print {$file} "# Generated by $PROGNAME $CVS_REVISION $TIMESTAMP\n";
	print {$file} "#\n";
	print {$file} "# Cron job file for primary server to $cron_job_type server via rsync.\n";
	print {$file} "#\n";
	print {$file} "30 3 * * * root (/usr/bin/test -e $cmd && $cmd $cmd_opts)\n";

	close($file);

	system("chown root:root $cron_job_path");
	system("chmod 644 $cron_job_path");
    }
    else {
	showerror("Can not make new cron job file: $cron_job_path");
	return(0);
    }

    return(1);
}


#
# Remove a cron job.
#
# Returns
#   1 on success
#   0 on error
#   
sub tfr_cron_job_rm
{
    my ($cron_job_type) = @_;

    my $cron_job_file = "";
    if ($cron_job_type eq $CRON_JOB_TYPE_SERVER) {
	$cron_job_file = $CRON_JOB_SERVER_PATH;
    }
    elsif ($cron_job_type eq $CRON_JOB_TYPE_CLOUD) {
	$cron_job_file = $CRON_JOB_CLOUD_PATH;
    }
    else {
	showerror("can't happen: unknown cron job type: $cron_job_type");
	return(0);
    }

    if (-e $cron_job_file) {
	unlink($cron_job_file);
	if (-e $cron_job_file) {
	    showerror("can't remove cron job file: $cron_job_file");
	    return(0);
	}
    }
    else {
	loginfo("cron job file does not exist, no need to remove: $cron_job_file");
    }

    return(1);
}


#
# generate the contents of the default config file.
#
# Returns
#   string with contents
#
sub tfr_generate_default_config_file
{
    my $config_text_header = $EMPTY_STR;

    $config_text_header  = "#\n";
    $config_text_header .= "# $PROGNAME Config File\n";
    $config_text_header .= "# Generated by $PROGNAME $CVS_REVISION $TIMESTAMP\n";
    $config_text_header .= "#\n";

    my $config_text = << 'END_CONFIG_TEXT';
#
# email=xxxxx
#
# When backups are complete, if the "email" attribute is given
# a value, than an email is sent to the recipient specified.
# To specify a list of more than one recipient,
# use multiple "email=" lines, one email address per line.
#
#email=user@somewhere.com
#email=user2@elsewhere.com

#
# emailserver=smtp.isp.com
#
# Instead of using the default system mail agent, to use
# a 3rd party mail facility like Gmail, or Yahoo, or
# a local ISP, use the "email_server", "email_username", and
# "email_password" attributes to define an account to use
# for sending email.
#
# NOTE that your password will be stored here in cleartext.
#
#email_server=smtp.google.com
#email_username=someone@gmail.com
#email_password=gmailpassword
#
# Use the sendmail program
#email_server=sendmail

#
# send-summary
#
# If set to true, send summary backup report if email configured
# Default: false.
#
#send-summary=true
#send-summary=false

#
# printer=xxxxx
#
# If printers are configured, the backup summary report
# will be output to all named printers.
# Also, if the backup class is "device" and there is no
# backup device discovered, and printers are configured,
# an error message will be output to all named printers.
# Multiple "printer=xxxxxxx" lines are allowed, in which case,
# results will be sent to multiple printers.
#
# "xxxxxx" represents the printer queue name, ie the
# "cups printer name".
#
#printer=printer11
#printer=order1

#
# exclude=/path/to/somewhere
# Exclude files or paths from the configured backups.
# Note that multiple "exclude=" lines are allowed, and results are passed
# directly into a tar "exclude" file (see tar man page.)
#
#exclude=/path/to/file
#backup-exclude=/path/to/file
#exclude=filename
#backup-exclude=filename

#
# restore-exclude=/path/to/directory
# restore-exclude=/path/to/file
# Multiple of these are allowed.
# Only used during restore.
# Note: if there were values specified on the command line via the
# "--restore-exclude=" option, then the values specified in the
# config file will be added to those specified on the command line.
#restore-exclude=/path/to/directory
#restore-exclude=/path/to/file
#restore-exclude=filename

#
# userfile=/path/to/somewhere
# userfile=/path/to/somewhere
# ...
# Include "user specified" files for backup into the "--backup=include"
# option.
# Note that multiple "include=" lines are allowed.
# Note that "exclude" rules (above) are ignored when these "userfiles" are
# backed up.
#
#userfile=/path/to/file
#userfile=filename

#
# Specify which device we will use to backup.
# Note that this device is normally auto-detected, however, by specifying a default,
# one could use "non-standard" devices such as external hard disks, USB thumb drives
# or even "loopback" image files.
#
# Use this option with care!
# Especially when working with removable media. If you specify some device here
# which turns out to be your primary partition, you will wind up wiping out your
# primary partition.
#
# To create an image file, some examples are below. Make sure to --format the device
# before use:
# dd if=/dev/zero of=/path/to/file.img bs=1M count=30000 #30 GB image file.
#
#device=/dev/scd0
#device=/path/to/file.img

#
# Identify the vendor and model of an external disk.  These are the
# strings that appear in the following files:
#
#   /sys/block/sd[a|b|c|d|e]/device/vendor
#   /sys/block/sd[a|b|c|d|e]/device/model
#
# Default values:
#device-vendor=WD
#device-model="My Passport"
#
# Example for other block devices:
#device-vendor=Seagate
#device-model="FreeAgent GoFlex"

# If set to true, run in debug mode
# Default: false.
#
#debugmode=true
#debugmode=false

# If set to true, then look for a device on the USB bus that
# has a file system label of "TFBUDSK-yyyymmdd".
# Default: false
#
#usb-device=true
#usb-device=false

# If set to false, then do not run the "harden_linux.pl" script
# even if it normally would be run.
# Default: true
#
#harden-linux=true
#harden-linux=false

#
# Hint the backup script that this is an "rti" system.
# This is normally auto-detected.
# Default: none
#
#rti=true
#rti=false

#
# Hint the backup script that this is a "daisy" system.
# This is normally auto-detected.
# Default: none
#
#daisy=true
#daisy=false

# 
# If set to true, do a backup to the cloud server
# Default: false
#
#cloud=false

#
# Specify the ip addr or hostname of the cloud server
# Default: yourcloudserver.com
#
#cloud-server=yourcloudserver.com

#
# Specify the ip addr of backup server
# Default: empty string
#
#rsync-server=192.168.1.22

#
# Specify the path to the destination directory
# Default: empty string
#
#rsync-dir=/path/to/directory

#
# Specify addtional arbitrary rsync options... one or
# more of these statements may be specified.
# Default: empty string
#
#rsync-option=--temp-dir=/usr2
#rsync-option=--progress


END_CONFIG_TEXT

    $config_text .= "#\n";
    $config_text .= "# Specify a timeout (in seconds) for rsync command\n";
    $config_text .= "# Timeout must be >= 0, 0 means no timeout\n";
    $config_text .= "# Default: $DEF_RSYNC_TIMEOUT\n";
    $config_text .= "#\n";
    $config_text .= "#rsync-timeout=$DEF_RSYNC_TIMEOUT\n";
    $config_text .= "\n";

    $config_text .= "#\n";
    $config_text .= "# Specify that backup retries should be done.\n";
    $config_text .= "# Default policy: do " . (($DEF_RETRY_BACKUP) ? "" : "not do ") . "retries\n";
    $config_text .= "#\n";
    $config_text .= "#retry-backup=" . (($DEF_RETRY_BACKUP) ? "true" : "false") . "\n";
    $config_text .= "\n";

    $config_text .= "#\n";
    $config_text .= "# If \"retry-backup=true\", specify number of retries.\n";
    $config_text .= "# Retries must be >= 0 and <= $MAX_RETRY_BACKUP_REPS\n";
    $config_text .= "# Default: $DEF_RETRY_BACKUP_REPS\n";
    $config_text .= "#\n";
    $config_text .= "#retry-reps=$DEF_RETRY_BACKUP_REPS\n";
    $config_text .= "\n";

    $config_text .= "#\n";
    $config_text .= "# If \"retry-backup=true\", specify seconds to wait between retries.\n";
    $config_text .= "# Wait must be >= 0 and <= $MAX_RETRY_BACKUP_WAIT\n";
    $config_text .= "# Default: $DEF_RETRY_BACKUP_WAIT\n";
    $config_text .= "#\n";
    $config_text .= "#retry-wait=$DEF_RETRY_BACKUP_WAIT\n";
    $config_text .= "\n";

    $config_text .= "#\n";
    $config_text .= "# Specify the network interface device name.\n";
    $config_text .= "# Default: $DEF_NETWORK_DEVICE\n";
    $config_text .= "#\n";
    $config_text .= "#network-device=$DEF_NETWORK_DEVICE\n";
    $config_text .= "\n";

    my $config_contents = $config_text_header . $config_text; 

    return($config_contents);
}


#
# Install the default config file.
#
# If the default config file does not already exist, generate and
# write config file to the standard place.  If one already exists,
# write it to the standard place with an extension of ".new".
#
# Returns
#   name of config file on success
#   empty string on error
#
sub tfr_install_default_config_file
{
    my ($conf_file) = @_;

    #
    # If config file exists, write a new one with ".new" suffix and
    # leave the old one in place.
    #
    if (-f $conf_file) {
	$conf_file .= ".new"
    }

    if (open(my $cfh, '>', $conf_file)) {
	my $config_text = tfr_generate_default_config_file();
	print {$cfh} $config_text;
	close($cfh);

	my $owner = tfr_tfsupport_account_name();
	my $group = tfr_pos_admin_group_name();
	system ("chown $owner:$group $conf_file");
	system ("chmod 640 $conf_file");
    }
    else {
	showerror("could not write to new default config file: $conf_file");
	$conf_file = $EMPTY_STR;
    }

    return($conf_file);
}


#
# top level sub for "--install-primary" and "--install-cloud"
#
# Install the tfrsync.pl configuration on the primary server.
# The installation steps consist of:
# 0) make the top level backup dir
# 1) make the transfer dir
# 2) make the pserver info dir
# 3) make the pserver cloister dir
# 4) make the users info dir
# 5) add a tfrsync account
# 6) generate the ssh key pair
# 7) copy the public key file to the transfer dir
# 8) install a cron job file
# 9) install the default tfrsync config file
#
# Returns
#   $EXIT_OK on success
#   non-zero on error
#
sub tfr_install_primary
{
    my ($account_name, $device_type) = @_;

    loginfo("installing $PROGNAME ($CVS_REVISION) on primary server");

    # make the top level backup dir
    my $top_level_bu_dir = tfrm_pathto_project_bu_dir();
    if (tfr_util_mkdir($top_level_bu_dir)) {
	loginfo("mkdir of top level tfrsync backup dir successful: $top_level_bu_dir");
    }
    else {
	showerror("could not make top level tfrsync backup dir: $top_level_bu_dir");
	return($EXIT_TOP_LEVEL_MKDIR);
    }

    # make the transfer dir
    my $pserver_xferdir_path = tfrm_pathto_pserver_xfer_dir();
    if (tfr_util_mkdir($pserver_xferdir_path)) {
	loginfo("mkdir of pserver transfer dir successful: $pserver_xferdir_path");
    }
    else {
	showerror("could not make pserver transfer dir: $pserver_xferdir_path");
	return($EXIT_XFERDIR_MKDIR);
    }

    # make the pserver info dir
    my $pserver_infodir_path = tfrm_pathto_pserver_info_dir();
    if (tfr_util_mkdir($pserver_infodir_path)) {
	loginfo("mkdir of pserver info dir successful: $pserver_infodir_path");
    }
    else {
	showerror("could not make pserver info dir: $pserver_infodir_path");
	return($EXIT_INFODIR_MKDIR);
    }

    # make the pserver cloister dir
    my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
    if (tfr_util_mkdir($pserver_cloister_dir_path)) {
	loginfo("mkdir of pserver clositer dir successful: $pserver_cloister_dir_path");
    }
    else {
	showerror("could not make pserver clositer dir: $pserver_cloister_dir_path");
	return($EXIT_INFODIR_MKDIR);
    }

    # make the users info dir
    my $users_infodir_path = tfrm_pathto_users_info_dir();
    if (tfr_util_mkdir($users_infodir_path)) {
	loginfo("mkdir of users info dir successful: $users_infodir_path");
    }
    else {
	logerror("could not make users info dir: $users_infodir_path");
	return($EXIT_USERSDIR_MKDIR);
    }

    # add "tfrsync" account if necessary
    if (tfr_accounts_verify_account($account_name)) {
	loginfo("account already exists: $account_name");
    }
    else {
	if (tfr_accounts_add_account($account_name, $DEF_RSYNC_ACCOUNT_FULL_NAME)) {
	    loginfo("added account: $account_name");
	}
	else {
	    showerror("error adding account: $account_name");
	    return($EXIT_RSYNC_ACCOUNT);
	}
    }

    # generate the password-less ssh key pair
    if (tfr_keys_generate_ssh_keys($account_name)) {
	loginfo("ssh keypair generated for account: $account_name");
    }
    else {
	showerror("error generating SSH keypair for account: $account_name");
	return($EXIT_SSH_GENERATE_KEYS);
    }

    # copy the public key file that was just generated for the "tfrsync" account
    # to the pserver transfer dir
    if (tfr_keys_copy_public_key_file($account_name, $pserver_xferdir_path)) {
	loginfo("public key for \"$account_name\" copied to: $pserver_xferdir_path");
    }
    else {
	showerror("error copying public key for \"$account_name\" to: $pserver_xferdir_path");
	return($EXIT_SSH_COPY_PUBLIC_KEY);
    }

    # install a cron job file
    my $cron_job_type = tfr_cron_job_type($device_type);
    if (tfr_cron_job_add($cron_job_type)) {
	loginfo("new cron job type installed: $cron_job_type");
    }
    else {
	showerror("error installing cron job type: $cron_job_type");
	return($EXIT_CRON_JOB_FILE);
    }

    # install the default config file - careful - the name of the
    # default config file might have changed, ie it might have a
    # ".new" suffix on it.
    my $conf_file_path = tfrm_pathto_def_pos_config_file();
    my $installed_conf_file_path = tfr_install_default_config_file($conf_file_path);
    if ($installed_conf_file_path) {
	loginfo("new default config file installed: $installed_conf_file_path");
    }
    else {
	showerror("error installing default config file: $conf_file_path");
	return($EXIT_DEF_CONFIG_FILE);
    }

    return($EXIT_OK);
}


#
# top level sub for "--uninstall-primary" and "--uninstall-cloud"
#
# Uninstall the tfrsync.pl configuration on the primary server.
# The un-installation steps consist of:
# 1) remove the cron job file
# 2) remove the xfer dir
# 3) delete the tfrsync account
#
# Returns
#   $EXIT_OK on success
#   non-zero exit status on error
#
sub tfr_uninstall_primary
{
    my ($rsync_account_name, $device_type) = @_;

    loginfo("uninstalling $PROGNAME ($CVS_REVISION) on primary server");

    # remove the cron job file
    my $cron_job_type = tfr_cron_job_type($device_type);
    if (tfr_cron_job_rm($cron_job_type)) {
	loginfo("cron job type removed: $cron_job_type");
    }
    else {
	showerror("error removing cron job type: $cron_job_type");
	return($EXIT_CRON_JOB_FILE);
    }

    # remove the pserver xfer dir
    my $pserver_xferdir_path = tfrm_pathto_pserver_xfer_dir();
    if (tfr_util_rmdir($pserver_xferdir_path)) {
	loginfo("production server xfer dir removed");
    }
    else {
	showerror("error removing production server xfer dir");
	return($EXIT_XFERDIR_RMDIR);
    }

    # remove the "tfrsync" account
    if (tfr_accounts_rm_account($rsync_account_name)) {
	loginfo("account removed: $rsync_account_name");
    }
    else {
	showerror("error removing account: $rsync_account_name");
	return($EXIT_RSYNC_ACCOUNT);
    }

    return($EXIT_OK);
}


#
# top level sub for "--info-primary" or "--info-cloud"
#
# Report info about the state of the tfrsync.pl installation
# on the primary server.
#
# 0) does the tfrsync account exist?
# 1) does the ssh private key exist?
# 2) does the ssh public key exist?
# 3) has the public key file been copied to the xfer dir of
#    the support accounts home dir?
# 4) has the cron job been installed?
# 5) install the default tfrsync.pl config file
#
sub tfr_info_primary
{
    my ($account_name) = @_;

    my @info_recs = ();

    my $info_line = "tfrsync account exists: ";
    if (tfr_accounts_exists($account_name)) {
	my $homedir_path = tfr_accounts_homedir($account_name);
	$info_line .= "yes: $homedir_path";
	push(@info_recs, $info_line);

	$info_line = "tfrsync private key exists: ";
	my $private_key_path = tfr_pathto_ssh_id_file($account_name);
	if (-e $private_key_path) {
	    $info_line .= "yes: $private_key_path";
	    push(@info_recs, $info_line);
	}
	else {
	    $info_line .= "no";
	    push(@info_recs, $info_line);
	}

	$info_line = "tfrsync public key exists: ";
	my $public_key_path = tfr_pathto_ssh_public_key_file($account_name);
	if (-e $private_key_path) {
	    $info_line .= "yes: $public_key_path";
	    push(@info_recs, $info_line);

	    $info_line = "tfrsync public key copied to xfer dir: ";
	    my $xferdir_path = tfrm_pathto_pserver_xfer_dir();
	    my $cached_key_path = $xferdir_path . '/' . $SSH_KEY_FILENAME_PUBLIC;
	    if (-e $cached_key_path) {
		$info_line .= "yes: $cached_key_path";
	    }
	    else {
		$info_line .= "no";
	    }
	    push(@info_recs, $info_line);
	}
	else {
	    $info_line .= "no";
	    push(@info_recs, $info_line);
	}
    }
    else {
	$info_line .= "no";
	push(@info_recs, $info_line);
    }

    foreach my $cron_job_type (@CRON_JOB_TYPES) {
	my $cron_job_path = tfr_cron_job_path($cron_job_type);
	$info_line = "\"$cron_job_type\" cron job installed: ";
	if (-e $cron_job_path) {
	    $info_line .= "yes: $cron_job_path";
	}
	else {
	    $info_line .= "no";
	}
	push(@info_recs, $info_line);
    }

    $info_line = "tfrsync.pl config file exists: ";
    my $config_file_path = tfrm_pathto_config_file();
    $info_line .= (-e $config_file_path) ? "yes: $config_file_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "tfrsync backup dir exists: ";
    my $bu_dir = tfrm_pathto_project_bu_dir();
    $info_line .= (-d $bu_dir) ? "yes: $bu_dir" : 'no';
    push(@info_recs, $info_line);

    $info_line = "production server info dir exists: ";
    my $pserver_info_dir_path = tfrm_pathto_pserver_info_dir();
    $info_line .= (-d $pserver_info_dir_path) ? "yes: $pserver_info_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "production server cloister dir exists: ";
    my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
    $info_line .= (-d $pserver_cloister_dir_path) ? "yes: $pserver_cloister_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "users info dir exists: ";
    my $users_info_dir = tfrm_pathto_users_info_dir();
    $info_line .= (-d $users_info_dir) ? "yes: $users_info_dir" : 'no';
    push(@info_recs, $info_line);

    my $longest_prefix = -1;
    foreach (@info_recs) {
	my $prefix_end = index($_, ':');
	if ($prefix_end > $longest_prefix) {
	    $longest_prefix = $prefix_end;
	}
    }

    foreach (@info_recs) {
	my $prefix_end = index($_, ':');
	my $leading_spaces = " " x ($longest_prefix - $prefix_end);
	my $formatted_line = $leading_spaces . $_;
	print "$formatted_line\n";
    }

    return($EXIT_OK);
}


#
# top level sub for "--install-secondary"
#
# Install the tfrsync.pl configuration on the backup server.
# The installation steps consist of:
# 1) make the top level backup dir
# 2) make the transfer dir
# 3) add the tfrsync account
# 4) get the public key from the production server
# 5) add an entry to the sudoers file for tfrsync
# 6) service httpd stop and chkconfig httpd off if RTI
# 7) service rti stop and chkconfig rti off if RTI
# 8) move /usr2/bbx/bin/doveserver.pl to /usr2/bbx/bin/doveserver.pl.save if RTI
# 9) service blm stop and chkconfig blm off if RTI
# 10) service bbj stop and chkconfig bbj off if RTI
#
# Returns
#   0 on success
#   non-zero exit status on error
#
sub tfr_install_secondary
{
    my ($rsync_account_name, $primary_server) = @_;

    loginfo("installing $PROGNAME ($CVS_REVISION) on backup server");

    # make the top level backup dir
    my $top_level_bu_dir = tfrm_pathto_project_bu_dir();
    if (tfr_util_mkdir($top_level_bu_dir)) {
	loginfo("mkdir of top level tfrsync backup dir successful: $top_level_bu_dir");
    }
    else {
	showerror("could not make top level tfrsync backup dir: $top_level_bu_dir");
	return($EXIT_TOP_LEVEL_MKDIR);
    }

    # make the transfer dir
    my $pserver_xferdir_path = tfrm_pathto_pserver_xfer_dir();
    if (tfr_util_mkdir($pserver_xferdir_path)) {
	loginfo("mkdir of pserver transfer dir successful: $pserver_xferdir_path");
    }
    else {
	showerror("could not make pserver transfer dir: $pserver_xferdir_path");
	return($EXIT_XFERDIR_MKDIR);
    }

    # add "tfrsync" account if necessary
    if (tfr_accounts_verify_account($rsync_account_name)) {
	loginfo("account already exists: $rsync_account_name");
    }
    else {
	if (tfr_accounts_add_account($rsync_account_name, $DEF_RSYNC_ACCOUNT_FULL_NAME)) {
	    loginfo("account added: $rsync_account_name");

	    my $account_sshdir_path = tfr_accounts_make_default_sshdir($rsync_account_name);
	    if ($account_sshdir_path) {
		loginfo("default ssh dir made for account: $rsync_account_name");
	    }
	    else {
		showerror("could not make default ssh dir for account: $rsync_account_name");
		return($EXIT_RSYNC_ACCOUNT);
	    }
	}
	else {
	    showerror("error adding account: $rsync_account_name");
	    return($EXIT_RSYNC_ACCOUNT);
	}
    }

    # get the public key from the primary server
    if (tfr_keys_get_primary_public_key($rsync_account_name, $primary_server)) {
	loginfo("primary public key obtained for account: $rsync_account_name");
    }
    else {
	showerror("error obtaining primary public key for account: $rsync_account_name");
	return($EXIT_SSH_GET_PUBLIC_KEY);
    }

    # add an entry to the sudoers file for the "tfrsync" account
    if (tfr_install_sudoers_config($rsync_account_name)) {
	loginfo("sudoers configuration updated for account: $rsync_account_name");
    }
    else {
	showerror("error configuring sudoers for account: $rsync_account_name");
	return($EXIT_SSH_SUDO_CONF);
    }

    my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
    if (tfr_util_mkdir($pserver_cloister_dir_path)) {
	loginfo("mkdir successful: $pserver_cloister_dir_path");
    }
    else {
	showerror("mkdir unsuccessful: $pserver_cloister_dir_path");
	return($EXIT_CLOISTERDIR_MKDIR);
    }

    #
    # make sure that all the RTI processes and services are shutdown and
    # will not restart on reboot and prevent Dove from running.
    #
    if ($RTI) {
	if (is_service_configured("rti")) {
	    system("/sbin/service rti stop");
	    system("/sbin/chkconfig rti off");
	}
	if (is_service_configured("httpd")) {
	    system("/sbin/service httpd stop");
	    system("/sbin/chkconfig httpd off");
	}

	# deal with Dove server script
	if (-e $RTI_DOVE_CMD) {
	    system("cp -p $RTI_DOVE_CMD $pserver_cloister_dir_path");
	    system("rm -f $RTI_DOVE_CMD");
	}

	if (is_service_configured("blm")) {
	    system("/sbin/service blm stop");
	    system("/sbin/chkconfig blm off");
	}

	if (is_service_configured("bbj")) {
	    system("/sbin/service bbj stop");
	    system("/sbin/chkconfig bbj off");
	}
    }

    return($EXIT_OK);
}


#
# top level sub for "--uninstall-secondary"
#
# Uninstall the tfrsync.pl configuration on the secondary server.
# 1) remove the "tfrsync" account from the sudoers config
# 2) remove the public key for the "tfrsync" account
# 3) remove the "tfrsync" account 
#
# Returns
#   $EXIT_OK on success
#   non-zero exit status on error
#
sub tfr_uninstall_secondary
{
    my ($rsync_account_name) = @_;

    loginfo("uninstalling $PROGNAME ($CVS_REVISION) from backup server");

    # remove the tfrsync account from the /etc/sudoers file
    if (tfr_install_sudoers_rm()) {
	loginfo("account removed from sudoers configuration: $rsync_account_name");
    }
    else {
	showerror("error removing account from sudoers configuration: $rsync_account_name");
	return($EXIT_SSH_SUDO_CONF);
    }

    # remove the tfrsync account - the public key removed as side effect
    if (tfr_accounts_rm_account($rsync_account_name)) {
	loginfo("account removed: $rsync_account_name");
    }
    else {
	showerror("error removing account: $rsync_account_name");
	return($EXIT_RSYNC_ACCOUNT);
    }

    # remove the pserver xfer dir
    my $pserver_xferdir_path = tfrm_pathto_pserver_xfer_dir();
    if (tfr_util_rmdir($pserver_xferdir_path)) {
	loginfo("production server xfer dir removed");
    }
    else {
	showerror("error removing production server xfer dir");
	return($EXIT_XFERDIR_RMDIR);
    }

    return($EXIT_OK);
}


#
# top level sub for "--info-secondary"
#
# Report info about the state of the tfrsync.pl installation
# on the backup server.
#
# 1) does the tfrsync account exist?
# 2) is the public key from the primary server in the tfrsync account's
#    default ssh dir authorized keys file?
# 3) is there an entry in the sudoers file for the tfrsync account?
#
sub tfr_info_secondary
{
    my ($rsync_account_name) = @_;

    my @info_recs = ();

    my $info_line = "tfrsync account exists: ";
    if (tfr_accounts_exists($rsync_account_name)) {
	my $homedir_path = tfr_accounts_homedir($rsync_account_name);
	$info_line .= "yes: $homedir_path";
    }
    else {
	$info_line .= "no";
    }
    push(@info_recs, $info_line);

    $info_line = "tfrsync public key obtained: ";
    my $xferdir_path = tfrm_pathto_pserver_xfer_dir();
    if ($xferdir_path) {
	my $cached_key_path = $xferdir_path . '/' . $SSH_KEY_FILENAME_PUBLIC;
	if (-e $cached_key_path) {
	    if (tfr_keys_find_key_in_auth_keys($rsync_account_name, $cached_key_path)) {
		my $auth_keys_path = tfr_pathto_ssh_auth_keys_file($rsync_account_name);
		$info_line .= "yes: $auth_keys_path";
	    }
	    else {
	    }
	}
    }
    if (index($info_line, "yes: ") == -1) {
	$info_line .= "no";
    }
    push(@info_recs, $info_line);

    $info_line = "tfrsync account given privledge: ";
    if (fgrep("/etc/sudoers", $rsync_account_name) == 0) {
	$info_line .= "yes: /etc/sudoers";
    }
    else {
	$info_line .= "no";
    }
    push(@info_recs, $info_line);

    $info_line = "tfrsync backup dir exists: ";
    my $rsync_bu_dir_path = tfrm_pathto_project_bu_dir();
    $info_line .= (-d $rsync_bu_dir_path) ? "yes: $rsync_bu_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "production server info dir exists: ";
    my $pserver_info_dir_path = tfrm_pathto_pserver_info_dir();
    $info_line .= (-d $pserver_info_dir_path) ? "yes: $pserver_info_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "production server cloister dir exists: ";
    my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
    $info_line .= (-d $pserver_cloister_dir_path) ? "yes: $pserver_cloister_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "users info dir exists: ";
    my $users_info_dir = tfrm_pathto_users_info_dir();
    $info_line .= (-d $users_info_dir) ? "yes: $users_info_dir" : 'no';
    push(@info_recs, $info_line);

    my $longest_prefix = -1;
    foreach (@info_recs) {
	my $prefix_end = index($_, ':');
	if ($prefix_end > $longest_prefix) {
	    $longest_prefix = $prefix_end;
	}
    }

    foreach (@info_recs) {
	my $prefix_end = index($_, ':');
	my $leading_spaces = " " x ($longest_prefix - $prefix_end);
	my $formatted_line = $leading_spaces . $_;
	print "$formatted_line\n";
    }

    return($EXIT_OK);
}


#
# Report info about the state of the tfrsync.pl installation
# on the cloud server.
#
# 1) what is the name of the tfrsync account?  and does it exist?
# 2) has the ssh key pair been generated?  and does the public key
#    in the tfrsync account .ssh dir match the one in tfsupport's $HOME?
# 3) what is the ip addr of the cloud server?
#
sub tfr_info_cloud
{
    my ($account_name) = @_;

    my @info_recs = ();
    my $info_line = $EMPTY_STR;

    $info_line = "cloud file server ip addr: ";
    $info_line .= ($CLOUD_SERVER) ? "yes: $CLOUD_SERVER" : 'unknown';
    push(@info_recs, $info_line);

    $info_line = "tfrsync account exists: ";
    if (tfr_accounts_exists($account_name)) {
	my $homedir_path = tfr_accounts_homedir($account_name);
	$info_line .= "yes: $homedir_path";
	push(@info_recs, $info_line);

	$info_line = "tfrsync private key exists: ";
	my $private_key_path = tfr_pathto_ssh_id_file($account_name);
	if (-e $private_key_path) {
	    $info_line .= "yes: $private_key_path";
	    push(@info_recs, $info_line);
	}
	else {
	    $info_line .= "no";
	    push(@info_recs, $info_line);
	}

	$info_line = "tfrsync public key exists: ";
	my $public_key_path = tfr_pathto_ssh_public_key_file($account_name);
	if (-e $private_key_path) {
	    $info_line .= "yes: $public_key_path";
	    push(@info_recs, $info_line);

	    $info_line = "tfrsync public key copied to xfer dir: ";
	    my $xferdir_path = tfrm_pathto_pserver_xfer_dir();
	    my $cached_key_path = $xferdir_path . '/' . $SSH_KEY_FILENAME_PUBLIC;
	    if (-e $cached_key_path) {
		$info_line .= "yes: $cached_key_path";
	    }
	    else {
		$info_line .= "no";
	    }
	    push(@info_recs, $info_line);
	}
	else {
	    $info_line .= "no";
	    push(@info_recs, $info_line);
	}
    }
    else {
	$info_line .= "no";
	push(@info_recs, $info_line);
    }

    foreach my $cron_job_type (@CRON_JOB_TYPES) {
	my $cron_job_path = tfr_cron_job_path($cron_job_type);
	$info_line = "\"$cron_job_type\" cron job installed: ";
	if (-e $cron_job_path) {
	    $info_line .= "yes: $cron_job_path";
	}
	else {
	    $info_line .= "no";
	}
	push(@info_recs, $info_line);
    }

    $info_line = "tfrsync.pl config file exists: ";
    my $config_file_path = tfrm_pathto_config_file();
    $info_line .= (-e $config_file_path) ? "yes: $config_file_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "tfrsync backup dir exists: ";
    my $bu_dir = tfrm_pathto_project_bu_dir();
    $info_line .= (-d $bu_dir) ? "yes: $bu_dir" : 'no';
    push(@info_recs, $info_line);

    $info_line = "production server info dir exists: ";
    my $pserver_info_dir_path = tfrm_pathto_pserver_info_dir();
    $info_line .= (-d $pserver_info_dir_path) ? "yes: $pserver_info_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "production server cloister dir exists: ";
    my $pserver_cloister_dir_path = tfrm_pathto_pserver_cloister_dir();
    $info_line .= (-d $pserver_cloister_dir_path) ? "yes: $pserver_cloister_dir_path" : 'no';
    push(@info_recs, $info_line);

    $info_line = "users info dir exists: ";
    my $users_info_dir = tfrm_pathto_users_info_dir();
    $info_line .= (-d $users_info_dir) ? "yes: $users_info_dir" : 'no';
    push(@info_recs, $info_line);

    my $longest_prefix = -1;
    foreach (@info_recs) {
	my $prefix_end = index($_, ':');
	if ($prefix_end > $longest_prefix) {
	    $longest_prefix = $prefix_end;
	}
    }

    foreach (@info_recs) {
	my $prefix_end = index($_, ':');
	my $leading_spaces = " " x ($longest_prefix - $prefix_end);
	my $formatted_line = $leading_spaces . $_;
	print "$formatted_line\n";
    }

    return($EXIT_OK);
}


sub tfr_report_configfile_entry
{
    my ($configfile_entry) = @_;

    loginfo("config attribute parsed: $configfile_entry");
    if ($REPORT_CONFIGFILE) {
	print "config attribute parsed: $configfile_entry\n";
    }

    return(1);
}


sub tfr_parse_conf_file
{
    my ($line) = @_;

    chomp($line);

    # skip empty lines, lines with only white space, and comment lines
    if ( ($line eq $EMPTY_STR) ||
	 ($line =~ /^\s+$/)      ||
	 ($line =~ /^\s*#/) ) {
	return(1);
    }

    # printer=cups_prn_name
    # Multiple of these are allowed.
    if ($line =~ /^\s*printer\s*=\s*([[:print:]]+)$/i) {
	    push(@PRINTERS, $1);
	    tfr_report_configfile_entry("--printer=$1");
    }

    # email=someone@foo.com
    # Multiple of these are allowed.
    if ($line =~ /^\s*email\s*=\s*([[:print:]]+)$/i) {
	    push(@EMAIL_RECIPIENTS, $1);
	    tfr_report_configfile_entry("--email=$1");
    }

    # If we want to try sending emails via, say, gmail or yahoo.
    if ($line =~ /^\s*email_server\s*=\s*([[:print:]]+)$/i) {
	    $EMAIL_SERVER = $1;
	    tfr_report_configfile_entry("email_server =$1");
    }
    if ($line =~ /^\s*email_username\s*=\s*([[:print:]]+)$/i) {
	    $EMAIL_USER = $1;
	    tfr_report_configfile_entry("email_username=$1");
    }
    if ($line =~ /^\s*email_password\s*=\s*([[:print:]]+)$/i) {
	    $EMAIL_PASS = $1;
	    tfr_report_configfile_entry("email_password=*****");
    }

    # send summary report via email
    # send-summary=1
    # send-summary=true
    # send-summary=yes
    # only used by backup function
    if ($line =~ /^\s*send-summary\s*=\s*([[:print:]]+)$/i) {
	    if ($1 =~ /[YyTt1]/) {
		    $SEND_SUMMARY = 1;
		    tfr_report_configfile_entry("--send-summary");
	    }
    }

    # exclude=/path/to/directory
    # exclude=/path/to/file
    # Multiple of these are allowed.
    # Only used during backup.
    if ($line =~ /^\s*exclude\s*=\s*([[:print:]]+)$/i) {
	    push(@EXCLUDES, $1);
	    tfr_report_configfile_entry("exclude=$1");
    }
    if ($line =~ /^\s*backup-exclude\s*=\s*([[:print:]]+)$/i) {
	    push(@EXCLUDES, $1);
	    tfr_report_configfile_entry("backup-exclude=$1");
    }

    # restore-exclude=/path/to/directory
    # restore-exclude=/path/to/file
    # Multiple of these are allowed.
    # Only used during restore.
    # Note: if there were values specified on the command line via the
    # "--restore-exclude=" option, then the values specified in the
    # config file will be added to those specified on the command line.
    if ($line =~ /^\s*restore-exclude\s*=\s*([[:print:]]+)$/i) {
	    push(@RESTORE_EXCLUDES, $1);
	    tfr_report_configfile_entry("restore-exclude=$1");
    }

    # userfile=/path/to/directory
    # userfile=/path/to/file
    # Multiple of these are allowed.
    if ($line =~ /^\s*userfile\s*=\s*([[:print:]]+)$/i) {
	    push(@USERFILES, $1);
	    tfr_report_configfile_entry("userfile=$1");
    }

    # Which backup device to use.
    # device=/dev/whatever
    if ($line =~ /^\s*device\s*=\s*([[:print:]]+)$/i) {
	    $DEVICE = $1;
	    tfr_report_configfile_entry("--device=$DEVICE");
    }

    # Specify vendor for external backup device.
    # The default value is:
    # device-vendor=WD
    if ($line =~ /^\s*device-vendor\s*=\s*([[:print:]]+)$/i) {
	    $DEVICE_VENDOR = $1;
	    $DEVICE_VENDOR =~ s/"//g;
	    tfr_report_configfile_entry("--device-vendor=$DEVICE_VENDOR");
    }


    # Specify model for external backup device.
    # The default value is:
    # device-model=My Passport
    if ($line =~ /^\s*device-model\s*=\s*([[:print:]]+)$/i) {
	    $DEVICE_MODEL = $1;
	    $DEVICE_MODEL =~ s/"//g;
	    tfr_report_configfile_entry("--device-model=$DEVICE_MODEL");
    }

    # cloud backup
    # cloud=1
    # cloud=True / cloud=true
    # cloud=Yes / cloud=yes
    if ($line =~ /^\s*cloud\s*=\s*([[:print:]]+)$/i) {
	    if ($1 =~ /[YyTt1]/) {
		    $CLOUD = 1;
		    tfr_report_configfile_entry("--cloud");
	    }
    }

    # Specify cloud server
    # cloud-server=server.ip.address or hostname
    if ($line =~ /^\s*cloud-server\s*=\s*([[:print:]]+)$/i) {
	    $CLOUD_SERVER = $1;
	    tfr_report_configfile_entry("--cloud-server=$CLOUD_SERVER");
    }

    # Specify backup server
    # rsync-server=server.ip.address or hostname
    if ($line =~ /^\s*rsync-server\s*=\s*([[:print:]]+)$/i) {
	    $RsyncServer = $1;
	    tfr_report_configfile_entry("--rsync-server=$RsyncServer");
    }

    # Specify rsync directory
    # rsync-dir=path
    if ($line =~ /^\s*rsync-dir\s*=\s*([[:print:]]+)$/i) {
	    $RsyncDir = $1;
	    tfr_report_configfile_entry("--rsync-dir=$RsyncDir");
    }

    # Specify rsync option
    # rsync-option=string
    if ($line =~ /^\s*rsync-option\s*=\s*([[:print:]]+)$/i) {
	    push(@RSYNC_OPTIONS, $1);
	    tfr_report_configfile_entry("--rsync-option=$1");
    }

    # Specify rsync timeout
    # rsync-timeout=n
    if ($line =~ /^\s*rsync-timeout\s*=\s*(\d+)$/i) {
	    $RSYNC_TIMEOUT = $1;
	    tfr_report_configfile_entry("--rsync-timeout=$1");
    }

    # Specify backup retries should be done.
    # retry-backup=true
    if ($line =~ /^\s*retry-backup\s*=\s*([[:print:]]+)$/i) {
	    if ($1 =~ /[YyTt1]/) {
		    $RetryBackupCLO = 1;
		    tfr_report_configfile_entry("--retry-backup");
	    }
    }

    # Specify backup retry reps
    # retry-reps=n
    if ($line =~ /^\s*retry-reps\s*=\s*(\d+)$/i) {
	    $RetryBackupReps = $1;
	    tfr_report_configfile_entry("--retry-reps=$1");
    }

    # Specify backup retry wait
    # retry-wait=n
    if ($line =~ /^\s*retry-wait\s*=\s*(\d+)$/i) {
	    $RetryBackupWait = $1;
	    tfr_report_configfile_entry("--retry-wait=$1");
    }

    # Specify network interface device name
    # network-device=string
    if ($line =~ /^\s*network-device\s*=\s*([[:print:]]+)$/i) {
	    $NetworkDeviceCLO = $1;
	    tfr_report_configfile_entry("--network-device=$NetworkDeviceCLO");
    }

    # Run in debug mode
    # debugmode=1
    # debugmode=True
    # debugmode=Yes
    if ($line =~ /^\s*debugmode\s*=\s*([[:print:]]+)$/i) {
	    if ($1 =~ /[YyTt1]/) {
		    $DEBUGMODE = 1;
		    tfr_report_configfile_entry("--debugmode");
	    }
    }

    # Use a USB device?
    # usb-device=1
    # usb-device=True / usb-device=true
    # usb-device=Yes / usb-device=yes
    if ($line =~ /^\s*usb-device\s*=\s*([[:print:]]+)$/i) {
	    if ($1 =~ /[YyTt1]/) {
		    $USB_DEVICE = 1;
		    tfr_report_configfile_entry("--usb-device");
	    }
    }

    # Run the harden_linux.pl script?
    # harden-linux=1
    # harden-linux=True / harden-linux=true
    # harden-linux=Yes / harden-linux=yes
    if ($line =~ /^\s*harden-linux\s*=\s*([[:print:]]+)$/i) {
	    my $config_entry = "--harden-linux";
	    if($1 =~ /[YyTt1]/) {
		$HARDEN_LINUX = 1;
	    }
	    else {
		$HARDEN_LINUX = 0;
		$config_entry = "--noharden-linux";
	    }
	    tfr_report_configfile_entry($config_entry);
    }

    # Is this an RTI system?
    # rti=1
    # rti=True / rti=true
    # rti=Yes / rti=yes
    if ($line =~ /^\s*rti\s*=\s*([[:print:]]+)$/i) {
	    if($1 =~ /[YyTt1]/) {
		    $RTI = 1;
		    tfr_report_configfile_entry("--rti");
	    }
    }

    # Is this a daisy system?
    # daisy=1
    # daisy=True / daisy=true
    # daisy=Yes / daisy=yes
    if ($line =~ /^\s*daisy\s*=\s*([[:print:]]+)$/i) {
	    if($1 =~ /[YyTt1]/) {
		    $DAISY = 1;
		    tfr_report_configfile_entry("--daisy");
	    }
    }

    # ignore anything else

    return(1);
}


sub tfr_read_conf_file
{
    my ($conf_file) = @_;

    my $rc = 1;

    if (-e $conf_file) {
	if (open(my $config, '<', $conf_file)) {
	    loginfo("config file opened for read: $conf_file");
	    while(my $line = <$config>) {
		unless (tfr_parse_conf_file($line)) {
		    logerror("unrecognized line in config file: $line");
		    $rc = 0;
		}
	    }
	    if ($DEBUGMODE) {
		loginfo("config file read completed: $conf_file");
	    }
	    close($config) || logerror("error closing config file: $conf_file");
	}
	else {
	    showerror("could not open config file: $conf_file");
	}
    }
    else {
	logerror("config file does not exist: $conf_file");
	$rc = 0;
    }

    return($rc);
}


#
# Read configuration.
#
# if specified config file is not the default config file
# then
#   read only from path specified
# else
#   read from default config file and
#   any config files in the config directory
#
# Returns
#   number of config files read successfully
#
sub tfr_read_configuration
{
    my ($config_file_path) = @_;

    my $conf_file_count = 0;
    my @conf_files = ();

    if ($config_file_path eq $EMPTY_STR) {
	logerror("Can't happen: config file path is the empty string");
	return($conf_file_count);
    }

    if (tfr_read_conf_file($config_file_path)) {
	$conf_file_count++;
    }
    else {
	logerror("error reading config file: $config_file_path");
    }

    my $def_pos_config_dir_path = tfrm_pathto_def_pos_config_dir();
    my @dotdir_conf_files = glob("$def_pos_config_dir_path/*$DEF_CONFIG_FILE_EXT");

    foreach my $conf_file (@dotdir_conf_files) {
	if (tfr_read_conf_file($conf_file)) {
	    $conf_file_count++;
	}
	else {
	    logerror("error reading config file: $conf_file");
	    last;
	}
    }

    return($conf_file_count);
}


#
# setup all the log files and their location
#
# There is a log file for each of the destination classes:
#
#   cloud, server, and device
#
# For these log files, they are cummulative on a daily basis.
#
# The summary log file and the debug log file are cummulative.
#
# A directory to contain the log files may be specified
# on the command line.  If not, a default location is
# log directory of the POS, either RTI or Daisy.
#
sub tfr_log_setup
{
    my ($logfile_dir) = @_;

    my $rc = 1;

    # if the location of the log directory was specified
    # on the command line, verify it's usable,
    # otherwise choose the appropriate location
    if ($logfile_dir) {
	if (-d $logfile_dir) {
	    if (is_input_insecure($logfile_dir)) {
		print {*STDERR} "specified directory for log files insecure: $logfile_dir\n";
		$LogfileDir = $DEF_ALT_LOGFILE_DIR;
	    }
	    else {
		$LogfileDir = $logfile_dir;
	    }
	}
	else {
	    print {*STDERR} "specified location for log files not a directory: $logfile_dir\n";
	    $LogfileDir = $DEF_ALT_LOGFILE_DIR;
	}
    }
    elsif ($RTI) {
	$LogfileDir = $RTI_LOGDIR;
    }
    elsif ($DAISY) {
	$LogfileDir = $DAISY_LOGDIR;
    }
    else {
	$LogfileDir = $DEF_ALT_LOGFILE_DIR;
    }

    # now choose the logfile name
    if ($CLOUD) {
	$LogfileName = $DEF_LOGFILE_CLOUD;
    }
    elsif ($SERVER) {
	$LogfileName = $DEF_LOGFILE_SERVER;
    }
    elsif ($DEVICE) {
	$LogfileName = $DEF_LOGFILE_DEVICE;
    }
    else {
	$LogfileName = $DEF_LOGFILE;
    }

    # get current value of time and use it to form each of the various
    # types of log file names.
    my @current_time = localtime(time);

    # form path for current log file
    $LogfileName = POSIX::strftime($LogfileName, @current_time);
    $LOGFILE = File::Spec->catfile($LogfileDir, $LogfileName);

    if (tfr_log_rotate($LOGFILE, @current_time)) {
	loginfo("log file rotated: $LOGFILE");
    }

    # form path for summary log file
    $SummaryLogfile = File::Spec->catfile($LogfileDir, $DEF_SUMMARY_LOGFILE);

    # form path for debug log file
    $DebugLogfile = File::Spec->catfile($LogfileDir, $DEF_DEBUG_LOGFILE);

    if ($VERBOSE) {
	print "logfile directory: $LogfileDir\n";
	print "  current logfile: $LOGFILE\n";
	print "  summary logfile: $SummaryLogfile\n";
	if ($DEBUGMODE) {
	    print "   debug logfile: $DebugLogfile\n";
	}
    }

    return($rc);
}


#
# conclude log file operations
#
# Returns
#   1 on success
#   0 on error
#
sub tfr_log_conclude
{
    my $rc = 1;

    my $owner = tfr_tfsupport_account_name();
    my $group = tfr_pos_group_name();

    my $logfile_path = tfrm_pathto_logfile();
    my $summary_log_path = tfrm_pathto_summary_logfile();
    my $debug_log_path = tfrm_pathto_debug_logfile();

    my @log_paths = ($logfile_path, $summary_log_path, $debug_log_path);

    for my $log_path (@log_paths) {
	if (-e $log_path) {
	    system("chown $owner:$group $log_path");
	    system("chmod ugo=rw $log_path");     # yes, perms wide open
	}
    }

    return($rc);
}


#
# if month number of current time > month number of the log file,
# rotate, ie re-init, log file.
#
# The "@current_time" arg is a list as returned by function localtime().
#
# Returns
#   1 if logfile rotated
#   0 if not
#
sub tfr_log_rotate
{
    my ($logfile_path, @current_time) = @_;

    my $rc = 0;

    my $current_mon = POSIX::strftime("%m", @current_time);

    if (-e $logfile_path) {
	my $st = File::stat::stat($logfile_path);
	my $logfile_mon = POSIX::strftime("%m", localtime($st->mtime));

	if ($current_mon > $logfile_mon) {
	    unlink($logfile_path);
	    system("touch $logfile_path");
	    system("chmod ugo=rw $logfile_path"); # yes, perms wide open
	    $rc = 1;
	}
    }

    return($rc);
}


sub log_file_lock
{
    my ($lfh) = @_;

    my $rc = 1;

    if ($DEBUGMODE) {
	loginfo("waiting up to these many seconds to obtain log file lock: $WAIT_FOR_LOG_FILE_LOCK");
    }
    my $fail_safe = 0;
    while ($fail_safe++ < $WAIT_FOR_LOG_FILE_LOCK) {
	if (flock($lfh, LOCK_EX|LOCK_NB)) {
	    if ($DEBUGMODE) {
		loginfo("obtained log file lock on iteration: $fail_safe");
	    }
	    seek($lfh, 0, SEEK_END);
	    last;
	}
	sleep(1);
    }
    if ($fail_safe >= $WAIT_FOR_LOG_FILE_LOCK) {
	$rc = 0;
    }

    return($rc);
}


sub log_file_unlock
{
    my ($lfh) = @_;

    my $rc = flock($lfh, LOCK_UN);

    return($rc);
}


#
# Change the location of the logfile - when doing a "--restore=all" or
# a "--restore=rti" on an RTI system, or a "--restore=daisy" on a Daisy
# system, the logfile location is going to be replaced.
#
# The only real choice for a new temporary location is /tmp.  After
# the restore is done, copy the temporary logfile back to the normal
# location.
#
sub log_change_location
{
    return if ($LOGFILE eq "");

    # First, make a temp logfile based on name of standard logfile
    my $logfile_basename = basename($LOGFILE, '.log');
    my $template = $logfile_basename . '-XXXXXXX';
    my ($fh, $tmp_logfile) = tempfile($template, SUFFIX => '.log', DIR => '/tmp');
    close($fh);
    loginfo("tempfile for changing log file location: $tmp_logfile");

    # save path to old logfile
    my $old_logfile_path = $LOGFILE;

    # switch to temp logfile after writing message to old logfile
    loginfo("#");
    loginfo("# Switching to temp logfile: $tmp_logfile");
    loginfo("#");
    $LOGFILE = $tmp_logfile;

    # write message to new temp logfile
    loginfo("#");
    loginfo("# Logfile location switched");
    loginfo("#");
    loginfo("#\tTemporary logfile: $LOGFILE");
    loginfo("#\tPrevious logfile: $old_logfile_path");

    return($old_logfile_path);
}

sub log_restore_location
{
    my ($orig_logfile_path) = @_;

    # save path to temp logfile
    my $tmp_logfile = $LOGFILE;

    # switch to standard logfile after writing message to temp logfile
    loginfo("#");
    loginfo("# Switching to original logfile: $orig_logfile_path");
    loginfo("#");
    $LOGFILE = $orig_logfile_path;

    # write status message to standard logfile
    loginfo("#");
    loginfo("# Logfile location switched");
    loginfo("#");
    loginfo("#\tCurrent logfile: $LOGFILE");
    loginfo("#\tTemporary logfile: $tmp_logfile");
    loginfo("#");

    # concatenate temp logfile to the standard log file
    loginfo("#");
    loginfo("# BEGIN including contents of temp logfile");
    loginfo("#");
    system("cat $tmp_logfile >> $LOGFILE");
    my $cat_status = $?;
    loginfo("#");
    loginfo("# END including contents of temp logfile");
    loginfo("#");

    if ($cat_status != 0) {
	# could not copy contents of temp logfile
	logerror("Concatenation of temp logfile to standard log file failed");
	loginfo("Contents of temp logfile preserved: $tmp_logfile");
	loginfo("Please remove when no longer needed");
    }
    else {
	# success, so rm previous logfile
	unlink "$tmp_logfile";
	loginfo("Temp logfile removed: $tmp_logfile");
    }

    return(1);
}


# Output to screen, and write info to logfile.
sub showinfo
{
    my ($message) = @_;

    print("$message\n");

    return(loginfo("<I>  $message"));
}


# Output to screen, and write error to logfile.
sub showerror
{
    my ($message) = @_;

    print("$message\n");

    return(loginfo("<E>  $message"));
}


# Write error to logfile and output to screen if verbose.
sub logerror
{
    my ($message) = @_;

    if($VERBOSE != 0) {
	print("$message\n");
    }

    return(loginfo("<E>  $message"));
}


# Write debug info to logfile and output to screen if verbose.
sub logdebug
{
    my ($message) = @_;

    if($VERBOSE != 0) {
	print("$message\n");
    }

    return(loginfo("<D>  $message"));
}


#
# Write message to logfile.
#
sub loginfo
{
    my ($message) = @_;

    my $timestamp = POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime(time()));

    if (open(my $log, '>>', $LOGFILE)) {
	print {$log} "$timestamp";
	if ( ($message !~ /<E>/) &&  ($message !~ /<D>/) &&  ($message !~ /<I>/) ) {
	    print {$log} " <I> ";
	}
	print {$log} " $message\n";
	close($log);

	# insurance that processes can access the log file
	system("chmod ugo=rw $LOGFILE");
    }
    else {
	print "$timestamp $message\n";
    }

    return(1);
}


sub debuglog
{
    my ($message) = @_;

    my $timestamp = POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime(time()));

    if (open(my $log, '>>', $DebugLogfile)) {
	print {$log} "$timestamp $message\n";
	close($log);
    }
    else {
	print "$timestamp $message\n";
    }

    return(1);
}


__END__

=pod

=head1 NAME

tfrsync.pl - OSTools script to synchronize files with a backup server or device.

=head1 VERSION

This documenation refers to version: $Revision: 1.156.2.6 $


=head1 SYNOPSIS

tfrsync.pl --help

tfrsync.pl --version

tfrsync.pl --install-primary (--server | --cloud) [--rsync-account=name] [--network-device=s]

tfrsync.pl --uninstall-primary [--rsync-account=name]

tfrsync.pl --info-primary [--rsync-account=name]

tfrsync.pl --install-secondary [--primary-server=addr] [--rsync-account=name]

tfrsync.pl --uninstall-secondary [--rsync-account=name]

tfrsync.pl --info-secondary [--rsync-account=name]

tfrsync.pl --install-cloud [--cloud-server=addr] [---rsync-account=name]

tfrsync.pl --uninstall-cloud [--cloud-server=addr] [---rsync-account=name]

tfrsync.pl --generate-permfiles

tfrsync.pl --upload-permfiles [--cloud-server=addr] [--rsync-account=name]

tfrsync.pl --download-permfiles [--cloud-server=addr] [--rsync-account=name]

tfrsync.pl --restore-from-permfiles [--cloud-server=addr] [--rsync-account=name]

tfrsync.pl --backup=type --cloud [--cloud-server=addr] [--rsync-account=name]

tfrsync.pl --backup=type --server [--rsync-server=addr] [--rsync-account=name]

tfrsync.pl --restore=type --cloud [--cloud-server=addr] [--rsync-account=name]

tfrsync.pl --restore=type --server [--rsync-server=addr] [--rsync-account=name]

tfrsync.pl --list=type

tfrsync.pl --mount [--device=s]

tfrsync.pl --unmount [--device=s]

tfrsync.pl --getinfo

tfrsync.pl --report-configfile

tfrsync.pl --gen-default-configfile

tfrsync.pl --format [--force]

tfrsync.pl --runtime-cleanup ( --cloud | --server )


=head1 OPTIONS

There are two types of options.
The first is essentially a command since without this type of option,
the script will do nothing.
The second type is a modifier to a command option;
it tailors the behavior of the command.


=head2 Commands

=over 4

=item B<--version>

Output the version number of the script and exit.

=item B<--help>

Output a usage help message and exit.

=item B<--install|install-primary>

Install the I<tfrsync.pl> script on the primary server and
perform other steps to prep the primary server.

=item B<--uninstall-primary>

Undo what was done to install C<tfrsync.pl> on the primary server.

=item B<--info-primary>

Report configuration of primary server.

=item B<--install-secondary>

Prepare the secondary server.

=item B<--uninstall-secondary>

Undo what was done to install C<tfrsync.pl> on the secondary server.

=item B<--info-secondary>

Report configuration of backup server.

=item B<--generate-permfiles>

Generate perm files for all backup types.
For all the files that make up a backup type,
the F<getfacl(1)> command is used to capture the file perms of
the files that make up the backup type.
It writes the captured info into a perm file corresponding to the backup type.
If an error occurs while iterating through the list of files for
any backup type, the error is logged but the generation will continue.
Thus, if there are any errors reported, the contents of the perm file
may not be complete.
An example of an error that may occur is one caused when there are
stale symlinks in the set of files of a backup type.

=item B<--backup=type>

Perform a backup of the specified I<type>.
The I<type> may be one or more of the following:

	all
	usr2
	daisy
	printconfigs
	rticonfigs
	daisyconfigs
	osconfigs
	netconfigs
	userconfigs
	userfiles
	logfiles
	posusersinfo
	pserverinfo
	pservercloister
	bbxd
	bbxps
	singlefile

Modifiers: B<--device=s>, B<--usb-device>, B<--backup-exclude=list>,
B<--rsync-server=addr>, B<--rsync-dir=path>,
B<--retry-backup>, B<--retry-reps=number>, B<--retry-wait=seconds>,
B<--singlefile=path>, and B<--send-summary>

=item B<--restore=type>

Restore files of specified C<type> from a backup.
The I<type> may be one or more of the following:

	all
	usr2
	daisy
	printconfigs
	rticonfigs
	daisyconfigs
	osconfigs
	netconfigs
	userconfigs
	userfiles
	logfiles
	bbxd
	bbxps
	singlefile

Modifiers: B<--device=s>, B<--usb-device>, B<--restore-exclude=list>,
B<--force>, B<--rootdir=s>, B<--[no]harden-linux>, B<--singlefile=path> and
B<--dry-run>

=item B<--list=type>

List the files on a backup server or device for the specified backup type
(see B<--backup=type> option.
If no backup type is specfied, the default is I<all>.
This option may not be used with the B<--cloud> option.

Modifiers: B<--device=s>, B<--usb-device>, B<--rsync-server=addr>, B<--rsync-dir=path>

=item B<--format>

Format a backup device.

=item B<--getinfo>

Get and report info about the backups on a backup device.

=item B<--mount>

Mount a backup device.

=item B<--unmount|--umount>

Unmount a backup device.

=item B<--runtime-cleanup>

If the script crashed rather than ending with a clean exit,
there possibly will be a process lock file and/or
a SSH tunnel socket left behind.
This command can be used to cleanup those dangling files.
For cloud, server, or device backup operations,
the process lock file will be removed.
For server or cloud, the SSH tunnel socket will be removed.

=item B<--report-configfile>

Parse the config file, and report it's contents.

=item B<--gen-default-configfile>

Generate a new default config file and
put it into the OSTools config directory.
If there is already an existing config file,
do not overwrite the old config file -
rather, add the extension F<.new> before installation.

=back

=head2 Command Modifiers

=over 4

=item B<--server>

Specify this option when performing a backup from
a primary server to a backup server.
Sets the default values of B<--rsync-server=s> and
B<--rsync-account=s> if not set.

=item B<--cloud>

Specify this option when performing a backup from
a primary server to a cloud server.
Sets the default values of B<--cloud-server=s> and B<--rsync-account=s>.

=item B<--cloud-server=addr>

Specify the FQDN or IP address of the cloud server.

=item B<--primary-server=addr>

Sets the hostname or IP address of the primary server.
May only be used with B<--install-secondary>.

=item B<--rsync-server=addr>

Specify the hostname or IP address of the rsync backup server.

=item B<--rsync-dir=path>

Specify the directory to write files to on the rsync server if
B<--rsync-server> is specified or
to a local file system if B<--rsync-server> is not specified.

=item B<--rsync-account=s>

Sets the name of the account on the rsync server.
May be used with B<--install-primary>, B<--uninstall-primary>,
B<--info-primary>, B<--install-secondary>, B<--uninstall-secondary>,
B<--info-secondary>, B<--install-cloud>, B<--uninstall-cloud>,
B<--upload-permfiles>, B<--download-permfiles>,
B<--restore-from-permfiles>, B<--backup=type>, and B<--restore=type>.

=item B<--rsync-trial>

Run the I<rsync> command in trial mode
(accomplished by adding the B<-n> option to the I<rsync> command
that gets generated).
This provides a method to determine what files will be synchronized
without actually synchronizing them.

=item B<--rsync-options=s>

Specify a list of one or more comma separated I<rsync> options to be
added to the I<rsync> command line.

=item B<--rsync-timeout=s>

All I<rsync> commands generated have the B<--timeout=s> option
with a default value of 600 seconds.
Specify this option to change that value.
A value of zero means no timeout.

=item B<--[no]rsync-nice>

Run the C<rsync> command with C<nice>; this is the default behavior.
Specify I<--norsync-nice> on the command line to run the C<rsync>
command without C<nice>.

=item B<--[no]rsync-metadata>

The default behavior when doing a backup to the "cloud" is to
generated metadata files for each of the backup types.
To keep from generating metadata files,
specify the "--norsync-metatdata" option on the command line.

=item B<--rsync-compression>

By default, there is no compression of files during the transfer
by rsync, ie the B<-z> option is not present on the F<rsync> command.
If the B<--rsync-compression> option is specified with the
B<--backup=type> option, then the B<-z> option will be added
to the F<rsync> command issued.
Note that adding this option can increase the time required
to perform a backup when backing up very large files.

=item B<--retry-backup>

If present, enables the backup retry policy.
Note that retries are only applicable for instances
when F<rsync(1)> returns a value of 30 (I/O error) or
255 (SSH connection error).

=item B<--retry-reps=number>

If present, sets the number of retries to "number".
This option is ignored unless B<--retry-backup> is also specified.
The value specified must be >= 0 and <= 10.
If B<--retry-backup> is specified and
B<--retry-reps=number> is not specified or
if B<--retry-reps=0> is specified,
then the default value of 3 retries is used.

=item B<--retry-wait=number>

If present, sets the retry wait time to "number" seconds.
This option is ignored unless B<--retry-backup> is also specified.
The value specified must be >= 0 and <= 600.
If B<--retry-backup> is specified and
B<--retry-wait=number> is not specified or
if B<--retry-wait=0> is specified,
then the default value of 120 seconds is used.

=item B<--singlefile=path>

Provides the method to specify one or more files to backup or restore.
The files are specified by a list of one or more comma separated paths.
This option may only be used in conjunction with the
B<--backup=type> or B<--restore=type> commands.

=item B<--restore-upgrade>

An option that may only be used when both B<--cloud> and
B<--restore=type> are specified, ie that a restore is being
done from a cloud server to a staged production server.
The set of backup files being restored may or may not be
from the same platform as the staged server, ie the backup files
may be from a RHEL5 system and the staged server may be a RHEL6 system.

=item B<--network-device=s>

This option allows the specification of a network interface device name,
eg "eth0" or "eth1", and
is only used to produce the contents of the
production server info file when performing a backup.
The default value is "eth0" so if you are using that interface
for your network tap, there is no need to specify this option.

=item B<--send-summary>

A summary report is sent to a list of one or more email addresses
if email is configured.

=item B<--rti>

Specify that the system is a RTI system.

=item B<--daisy>

Specify that the system is a Daisy system.

=item B<--email=recipients>

Specifies a list of email addresses.

=item B<--printer=names>

Specifies a comma separated list of printer names.
If printers are configured, the backup summary report
will be output to all named printers.
Also, if the backup class is "device" and there is no
backup device discovered, and printers are configured,
an error message will be output to all named printers.

=item B<--rootdir=path>

Specifies the destination directory for restore.

=item B<--configfile=path>

Specifies the path to the config file.

=item B<--logfile=path>

Specifies the path to the logfile.

=item B<--backup-exclude=list>

Specifies a list of one or more comma separated files and/or directories
to exclude from a backup.

=item B<--restore-exclude=list>

Specifies a list of one ore more comma separated files and/or directories
to exclude from a restore.

=item B<--device=path>

Specifies the path to the device special file for the backup device.
It can also have the special value of I<show-only> whose
meaning is described under the B<--show-only> option description.

Each backup device is classified as to type.
There are several different device types supported:

 1. "passport"    - a Western Digital Passport USB disk
 2. "rev"         - an IOmega Rev drive
 3. "usb"         - a USB disk with Teleflora label
 4. "image"       - an image file
 5. "server"      - ip address of backup server
 6. "file system" - locally mounted file system
 7. "block"       - either "passport", "rev", or "usb"
 8. "show-only"   - special device, same thing as "--show-only"

=item B<--usb-device>

Use a disk plugged into the USB bus which has been
formatted with B<--format> as the backup device.

=item B<--dry-run>

Report what an operation would do but don't actually do it.

=item B<--show-only>

Change the behavior of B<--backup> to only show the filenames
of the backup type not actually back up any files.
Another way to think about this option is that it is an alias
for B<--device=show-only>, ie you are specifying a
special device named I<show-only> which does not backup
any data but rather reports the filenames to be backed up.

=item B<--verbose>

Report extra information.

=item B<--[no]harden-linux>

Run (or don't run) the I<harden_linux.pl> script.
The B<--harden-linux|--noharden-linux> command line option provides
a way to specify whether or not the I<harden_linux.pl> script
should be run by the I<tfrsync.pl> script.
The default behavior for I<tfrsync.pl> is to run I<harden_linux.pl>
after performing any of the following restore types:
"all", "rticonfigs", "daisy", "daisyconfigs", "osconfigs" and "netconfigs".
The I<harden_linux.pl> script will only be run once after all restores are finished.
To prevent I<harden_linux.pl> from running, specify the following option:
B<--noharden-linux>.

=back


=head1 DESCRIPTION

The I<tfrsync.pl> script may be used to synchronize data
from a Teleflora RTI or Daisy Point of Sale server,
referred to below as the "primary server", to a cloud server,
a backup server, a backup device, or a local file system.
It is essentially a front end to the I<rsync(1)> command which
does the real work of reading and writing the data.
For maximum versatility, there are many options and
many ways that the script can be used.

Since the backup strategy is one based on I<rsync(1)>,
implied is that the first backup performed copies all files of interest
from the primary server to the backup destination and
could take a significant amount of time depending
on the characteristics of the destination.
For example, if there is a 25 GB of data to backup,
if the distination is a cloud server, and
if the upload bandwidth to the cloud server is typical,
it should take 10 hours or less.
Subsequent backups are incremental and only the files that have changed
are copied to the backup destination.
This results in a significant savings in time,
with a typical installation taking 30 minutes and usually less.


=head2 Installation

Before I<tfrsync.pl> can be used on either the primary server or
the backup server, it must be installed.
The I<tfrsync.pl> script itself will have been installed when
the OSTools package was installed.
However, the I<tfrsync.pl> must install a framework of directories and
files in order to accomplish it's job.
This installation must be performed on both the primary server and
the backup server separately.
The sections which follow outline what is done during primary and
backup server installations.


=head2 Installation on the Primary Server

The B<--install-primary> command line option performs all the steps
necessary to install the I<tfrsync.pl> script onto a RTI production server,
aka a primary server.

The OSTools installation process will already have copied the
I<tfrsync.pl> script to the OSTools F<bin> directory and
made a symlink from the RTI F<bin> directory to it which
makes it available from the command line for the I<tfsupport> account.

The steps performed during the installation of the primary server
are outlined below:

=over 4

=item 1.

make the I<tfrsync> backup directory if it does not exist.
For a RTI system, this is F</usr2/tfrsync>, and
for a Daisy system, this is F</d/tfrsync>.

=item 2.

make the primary server transfer directory if it does not exist.
This directory is located in the home directory of the I<tfsupport>
account and is named F<pserver_xfer.d>.
The transfer directory is used to hold files that are to be
copied from the primary server to the backup server.

=item 3.

make the primary server info directory if it does not exist.
This directory is located in the I<tfrsync> backup directory and
is named F<pserver_info.d>.
The primary server info directory is the location of the
production server info file which is named F<pserver_info.txt>.
The production server info file is generated each time
a backup transaction occurs.

=item 4.

make the primary server cloister directory if it does not exist.
This directory is located in the I<tfrsync> backup directory and
is named F<pserver_cloister.d>.
The cloister directory is used to hold backup copies of
certain files that need to be backed up but need to be
segregated into a special location rather than be copied
"in place" like the bulk of the files to be backed up.
Examples of such files are F</var/spool/cron> and F</etc/cron.d>.
These files need segregation because if they are copied "in place"
to the backup server, they would cause behavior on the backup server
that would not be wanted, ie they would cause cron jobs to run.

=item 5.

make the users info directory if it does not exist.
This directory is located in the I<tfrsync> backup directory and
is named F<users_info.d>.
The users info directory is used to hold files containing info
about the POS users on the primary server.
These files are copied to the backup server and are required
for transforming a backup server into a primary server.

=item 6.

if the I<tfrsync> account does not exist, add the I<tfrsync> account.

=item 7.

generate a password-less SSH key-pair for the I<tfrsyc> account and
copy the public SSH key file that was just generated
to the primary server transfer directory.
The name of the public key file is F<id_rsa.pub>.

=item 8.

add a new cron job file named F<tfrsync> to the directory
F</etc/cron.d>.  If there is an existing cron job file there,
then the cron job file is placed in the OSTools config
directory instead.

=item 9.

generate a new config file and put it into the OSTools config directory.
If there is already an existing config file for I<tfrsync.pl>,
then do not overwrite the old config file - rather, put the new config file
into the OSTools config dir with the extension F<.new>.

=back


=head2 Installation on the Backup Server

In addition to installation of I<tfrsync.pl> on the RTI primary server,
there are several installation steps that must also be performed
on the backup server, aka secondary server.
To install I<tfrsync.pl> on the backup server,
first install the OSTools package on the secondary server.
Then, run the I<tfrsync.pl> script on the backup server
specifying the B<--install-secondary> command line option.
The B<--primary-server=s> may also be specified with the
B<--install-secondary> option;
it is used to specify the hostname or IP address of the primary server.
If the B<--primary-server=s> option is not specifed, then
the default value for the IP address of the primary server is used.
The default value is 192.168.1.21.
The following steps are performed:

=over 4

=item 1.

the I<tfrsync> account is added if it does not exist

=item 2.

the I<tfrsync> account name is added to the F</etc/sudoers> file
to allow the I<tfrsync> account to run the C<rsync> command as root
without being prompted for a password.
This is accomplished by adding the I<tfrsync> account name to the
OSTools I<harden_linux.pl> config file and then running the
C<harden_linux.pl --sudo> command.

=item 3.

the public key of the I<tfrsync> account on the primary server is
added to the F<.ssh/authorized_keys> file of the
I<tfrsync> account on the backup server

=item 4.

service httpd stop and chkconfig httpd off

=item 5.

service rti stop and chkconfig rti off

=item 6.

move /usr2/bbx/bin/doveserver.pl to /usr2/bbx/bin/doveserver.pl.save

=item 7.

service blm stop and chkconfig blm off

=item 8.

service bbj stop and chkconfig bbj off

=back


=head2 Uninstall on the Backup Server

The B<--uninstall-secondary> command line option
undoes what was done to install the C<tfrsync.pl> script
on the secondary server.

=over 4

=item 1.

the I<tfrsync> account name is removed from the C</etc/sudoers> file
by removing the C<harden_linux.pl> sudoers content file from the
C<harden_linux.pl> sudoers config directory,
C</d/ostools/config/sudoers.d> on Daisy systems and
C</usr2/ostools/config/suders.d> on RTI systems, and
then running the C<harden_linux.pl --sudo> command.

=item 2.

the public key of the I<tfrsync> account on the primary server is
removed from the F<~tfrsync/.ssh/authorized_keys> file on the
backup server.

=item 3.

the I<tfrsync> account is removed from the backup server.

=back


=head2 Backup Retries

When doing a backup to a cloud server, if the Internet connection
is inconsistent and unreliable, the rsync command can fail with a timeout error.
For systems experiencing these conditions, the command line options
B<--retry-backup>, B<--retry-reps=n> and B<--retry-wait=secs> can be
very useful.
When B<--retry-backup> is specified, if there is an rsync timeout error,
the entire backup operation is retried one or more times with a wait
between each retry.
The default number of times to retry is 3.
The default time to wait between retries is 2 minutes (120 seconds).
The B<--retry-reps=n> and B<--retry-wait=secs> allows you to tune
the retry and wait values to best suite your situation.
These options are also supported in the config file so they
can be specified on the command line as well as the config file.


=head2 Rsync of Summary Log File

After a backup operation is complete, ie after all
changed files have been written to the backup device or
to the backup server or to the cloud server,
a summary log file is written locally to the POS log directory.
Thus, this summary log file will not be transferred to the backup device until
the next time a backup operation is performed.
In order to backup the summary log file and
to have convenient access to this file for monitoring and reporting,
a resync of the summary log file is performed
after the backup operation is complete and
the summary log file has been written.


=head2 Config File

The F<tfrsync.pl> script supports one or more config files.
The default config file name and location is specified in
the FILES section below.
There are many configuration attributes that can be set
in the config file.
Refer to the comments in the generated default config file
for a list of attributes, their meaning, and their values.


=head2 Production Server Info File

The production/primary server info file, aka pserver info file,
contains key information about the production server.
It is generated each time a backup is performed and
is copied to the backup server, or cloud server, or device
each time there is a backup.

The name of the pserver info file is F<pserver_info.txt> and
is located in the F</usr2/tfrsync/pserver_info.d> directory
on an RTI system and in the F</d/tfrsync/pserver_info.d>
directory on a Daisy system.
The file is a simple ASCII text file,
with each line consisting of an attribute/value pair,
separated by an EQUALS SIGN.

The pserver info file contains the following attributes;

=over 4

=item platform

The value is the name of the platform, either
"RHEL5", or "RHEL6".

=item hostname

The value is the hostname of the production server. 

=item ipaddr

The IP address of the production server

=item netmask

The netmask of the production server

=item gateway

The IP address of the "gateway", ie the router.

=back

As a concrete example, if the production server was a
Daisy "RHEL5" system,
the pserver info file would look similar to the following,
with values adjusted as appropriate:

 platform=RHEL5
 hostname=77777700-tsrvr
 ipaddr=192.168.1.21
 netmask=255.255.255.0
 gateway=192.168.1.1


=head2 Backup Summary Log File

The backup summary log file contains an entry for each execution
of the F<tfrsync.pl> script.
Each entry is about 750 bytes long and is a short summary of the backup result.
This file grows without bound, but due to the relatively small entry size and
frequency of execution, it should not be a factor in disk space usage.
A future update will add this file to the system F<logrotate(8)> policy when
installed on the primary server.
Each entry in the summary log file consists of a record of 17 lines,
one line per field, in the format oulined below.
If the value of an entry is not applicable to the type of backup being performed,
the value of the field will be the string "NA".

=over 4

=item separator line

A line of 80 "=" chars.

=item script name

The file name of the script, F<tfrsync.pl>.

=item script version

The CVS revision number of the script.

=item command line

The command line used to invoke the script.

=item execution start time

The execution start time in the format of "YMD-HMS", which
for example, Jan 27, 2014 at 10:40 and 24 seconds
would be "20150127-104024".

=item execution stop time

The execution stop time in the format of "YMD-HMS", which
for example, Jan 27, 2014 at 10:40 and 24 seconds
would be "20150127-104024".

=item execution duration

The length of time for the execution in the format of "H:M:S", which
for example, a duration of 0 hours, 10 minutes, and 27 seconds
would be "00:10:27".

=item device type

The backup device type, would will usually be
"cloud", "server", "passport", or "usb".

=item result description

The english equivaliant of the exit status.

=item rsync exit status

The exit status reported by the F<rsync> command used
to perform the backup.

=item rsync backup retries

The number of times that the F<rsync> command was retried.

=item rsync warnings

A list of one or more rsync exit status values that are
considered to  be just warnings.
If there are no warnings, then the value is just "0".

=item bytes written

The total number of bytes written by rsync.

=item server IP addr

The hostname or IP address of the cloud server or the backup server.

=item path on server

The value of the B<--rsync-dir> option if it was specified.
Not applicable for "cloud" backup.

=item device file path

The path of the device file for the backup device, which,
for example, might be F</dev/sdb1> for a Passport device.
Not applicable for "cloud" backup.

=item backup device capacity

The total capacity of the backup device.
Not applicable for "cloud" backup.

=item backup device available space

The available space left on the backup device.
Not applicable for "cloud" backup.

=item separator line

A line of 80 "=" chars.

=back

Here is an example of the contents of a summary log file:

 ================================================================================
     PROGRAM: tfrsync.pl
     VERSION: $Revision: 1.156.2.6 $
     COMMAND: /home/tfsupport/tfrsync.pl --server --backup=osconfigs
       BEGIN: 20150210-133052
         END: 20150210-133103
    DURATION: 00:00:11
      DEVICE: server
      RESULT: Exit OK
       RSYNC: 0
     RETRIES: 0
    WARNINGS: 0
  BYTES SENT: 374.17KB
      SERVER: 192.168.2.31
        PATH: /tmp
 DEVICE FILE: NA
    CAPACITY: NA
   AVAILABLE: NA
 ================================================================================


=head1 EXAMPLES

To install F<tfrsync.pl> on a Teleflora POS system and
configure it as the primary server, enter the following:

 sudo tfrsync.pl --server --install-primary

To get info about the installation on the primary server, you can
enter the following:

 sudo tfrsync.pl --info-primary

To install F<tfrsync.pl> on a Teleflora POS system and
configure it as the secondary server, enter the following:

 sudo tfrsync.pl --install-secondary --primary-server=192.168.1.21

On an RTI system,
to backup all files in all backup types to the backup server at
IP addr 192.168.1.22, in the directory F</usr2/tfrsync>, enter the following:

 sudo tfrsync.pl --server --backup=all --rsync-server=192.168.1.22

To display the filenames contained in the backup type C<netconfigs>,
enter the following:

 sudo tfrsync.pl --server --backup=netconfigs --show-only

To perform a backup operation to a cloud server with retries,
with the number of retries 5 and the time of 3 minutes to wait between each retry,
enter the following:

 sudo tfrsync.pl --cloud --backup=all --retry-backup --retry-reps=5 --retry-wait=180

To backup a single file, say F</etc/motd>, from the production server
to the backup server and put it in F</tmp> on the backup server,
enter the following:

 sudo tfrsync.pl --server --singlefile=/etc/motd --backup=singlefile --rsync-dir=/tmp

To restore the single file F</tmp/etc/printcap> from the backup server and
put it in the F</tmp> directory on the production server,
enter the following:

 sudo tfrsync.pl --server --singlefile=/etc/printcap --restore=singlefile \
    --rsync-dir=/tmp --rootdir=/tmp


=head1 FILES

=over 4

=item F</usr2/tfrsync>

the top level I<tfrsync> backup directory for a RTI system

=item F</d/tfrsync>

the top level I<tfrsync> backup directory for a Daisy system

=item F</home/tfrsync>

the home directory of the I<tfrsync> account

=item F</home/tfsupport>

the home directory of the Teleflora support account

=item F</home/tfsupport/pserver_xfer.d>

the primary server transfer directory

=item F<pserver_info.d>

the primary server info directory and is located
in I<tfrsync> backup directory

=item F<pserver_info.txt>

the production server info file which is located in the
primary server info directory.
The production server info file is generated each time
a backup transaction occurs.

=item F</usr2/tfrsync/users_info.d/rti_users_listing.txt>

The RTI users listing file.

=item F</usr2/tfrsync/users_info.d/rti_users_shadow.txt>

The shadow file entries for the users in the RTI users listing file.

=item F</d/tfrsync/users_info.d/daisy_users_listing.txt>

The Daisy users listing file.

=item F</d/tfrsync/users_info.d/daisy_users_shadow.txt>

The shadow file entries for the users in the Daisy users listing file.

=item F<pserver_cloister.d>

the primary server cloister directory is located in the
I<tfrsync> backup directory

=item F<id_rsa.pub>

The name of the public SSH key file

=item F</etc/cron.d/tfrsync>

the I<tfrsync> cron job file

=item F<tfrsync-server-Day_nn.log>

The name of the log file for destination class "server".
The string I<nn> is replaced by the zero filled, two digit
day of the month.

=item F<tfrsync-cloud-Day_nn.log>

The name of the log file for destination class "cloud".
The string I<nn> is replaced by the zero filled, two digit
day of the month.

=item F<tfrsync-device-Day_nn.log>

The name of the log file for destination class "device".
The string I<nn> is replaced by the zero filled, two digit
day of the month.

=item F<tfrsync-summary.log>

The summary log file contains an entry for each execution
of the F<tfrsync.pl> script with a short summary of the result.

=item F</usr2/bbx/log> 

On an RTI system, the location of the log files.
If this directory does not exist, the log files will be
put in F</tmp>.

=item F</d/daisy/log>

On a Daisy system, the location of the log files.
If this directory does not exist, the log files will be
put in F</tmp>.

=item F</usr2/ostools/config>

The location of the default config file for an RTI system.

=item F</d/ostools/config>

The location of the default config file for a Daisy system.

=item F<tfrsync.conf>

The name of the default config file.

=item F</usr2/ostools/config/tfrsync.d>

The location of optional custom config files for an RTI system.

=item F</d/ostools/config/tfrsync.d>

The location of optional custom config files for a Daisy system.

=item F</mnt/backups>

Mount point for backup devices.

=item F</etc/redhat-release>

Contents determines OS type.

=item F</sys/block/{sda,sdb,sdc,sde,sdd}/device/vendor>

This file contains the vendor string for the block device, ie disk,
that has special device file "/dev/sda", or "/dev/sdb", etc.

=item F</sys/block/{sda,sdb,sdc,sde,sdd}/device/model>

This file contains the model string for the block device, ie disk,
that has special device file "/dev/sda", or "/dev/sdb", etc.

=item F</usr2/tfrsync/$backuptype-perms.txt>

The metadata files generated on a RTI system when doing a backup to the cloud.
See the output of F<tfrsync.pl --help> for a list of backup type values
that "$backuptype" may take.

=item F</d/tfrsync/$backuptype-perms.txt>

The metadata files generated on a Daisy sysstem when doing a backup to the cloud.
See the output of F<tfrsync.pl --help> for a list of backup type values
that "$backuptype" may take.

=item F</var/lock/tfrsync-server.lock>

=item F</var/lock/tfrsync-cloud.lock>

=item F</var/lock/tfrsync-device.lock>

The process lock file.
There is a separate process lock file for each of the device types.
The process lock file path will be one of the paths above,
for either the "server", "cloud", or "device",
which corresponds to each of the device types.
When the script starts, an attempt is made to acquire the lock file
corresponding to the device type;
if the lock file already exists when this check is made, then
it indicates that an existing process has the lock for that device type and
thus, a new process is not allowed to proceed. 

=item F</var/run/tfrsync-server.socket>

=item F</var/run/tfrsync-cloud.socket>

The SSH tunnel socket.
There is a separate SSH tunnel socket for each of the destination classes.
For the backup destination class "server",
the name of the socket is F<tfrsync-server.socket>.
For the "cloud" destination class, the name is F<tfrsync-cloud.socket>.
When a backup or restore operation is performed,
an attempt is made to establish an SSH tunnel on a socket at the appropriate path.
If the socket already exists, it indicates that a previous process
used the socket for a connection to the remote device and crashed
without cleaning up the socket;
the B<--runtime-cleanup> command can be used to cleanup stale sockets.
If the SSH command returns an exit status of 255,
the script makes 3 attempts to open the socket before
returning an error.

=item F</tmp/tfrsync-rsyncstats-XXXXXXX>

A temp file to hold the stats generated during a successful rsync command.
The stats are compiled immedately after a rsync command completes and
then the temp file is deleted.

=back


=head1 DIAGNOSTICS

=over 4

=item Exit status 0 ($EXIT_OK)

Successful completion.

=item Exit status 1 ($EXIT_COMMAND_LINE)

In general, there was an issue with the syntax of the command line.

=item Exit status 2 ($EXIT_MUST_BE_ROOT)

The script must run as root or with sudo(1).

=item Exit status 3 ($EXIT_ROOTDIR)

The directory specified for B<--rootdir> does not exist.

=item Exit status 4 ($EXIT_TOOLSDIR)

The OSTools directory does not exist.

=item Exit status 5 ($EXIT_BACKUP_DEVICE_NOT_FOUND)

A backup device of any kind was not found.

=item Exit status 6 ($EXIT_USB_DEVICE_NOT_FOUND)

The B<--usb-device> option was specified but a USB backup device
was not found.

=item Exit status 7 ($EXIT_BACKUP_TYPE)

The backup type specified with B<--backup> is not supported.

=item Exit status 8 ($EXIT_MOUNT_ERROR)

An error occurred when trying to mount a backup device that
was of type block device or image file.

=item Exit status 9 ($EXIT_RESTORE)

An unsupported restore type was specified on the command line.

=item Exit status 10 ($EXIT_DEVICE_NOT_SPECIFIED)

A backup device was not specified nor could be found

=item Exit status 11 ($EXIT_DEVICE_VERIFY)

The specified backup device is either not a block device or
is not an image file of minimum size.

=item Exit status 12 ($EXIT_USB_DEVICE_UNSUPPORTED)

USB devices other than WD Passports are not supported on RHEL4.

=item Exit status 13 ($EXIT_CRON_JOB_FILE)

An error occurred when installing the cron job file.

=item Exit status 14 ($EXIT_DEF_CONFIG_FILE)

An error occurred when installing the default config file.

=item Exit status 15 ($EXIT_FORMAT)

An error occurred when formatting the backup device.

=item Exit status 16 ($EXIT_LIST_UNSUP)

An unsupported list type was specified on the command line.

=item Exit status 17 ($EXIT_LIST)

The files of the specified backup type could not be listed.

=item Exit status 18 ($EXIT_SIGINT)

Script received interrupt signal during a rsync transaction.

=item Exit status 19 ($EXIT_RSYNC_ACCOUNT)

An error occurred either when making the I<tfrsync> account
while installing or
when removing the I<tfrsync> account while uninstalling.

=item Exit status 20 ($EXIT_SSH_GENERATE_KEYS)

An error occurred when generating the RSA keypair for
the account used to run the rsync command.

=item Exit status 21 ($EXIT_SSH_GET_PUBLIC_KEY)

An error occurred when getting the public key for an
C<rsync> account on the primary server.

=item Exit status 22 ($EXIT_SSH_GET_PRIVATE_KEY)

An error occurred when getting the private key for an
C<rsync> account on the primary server.

=item Exit status 23 ($EXIT_SSH_SUDO_CONF)

An error occurred configuring the F</etc/sudoers> file
to allow an C<rsync> account to run I<rsync> command via I<sudo>; 
this is accomplished not by directly editing F</etc/sudoers>, but
rather by using the mechanism for doing this provided in the
I<harden_linux.pl> config file.

=item Exit status 24 ($EXIT_SSH_TUNNEL_OPEN)

An error occurred opening SSH tunnel.

=item Exit status 25 ($EXIT_SSH_TUNNEL_CLOSE)

An error occurred closing SSH tunnel.

=item Exit status 26 ($EXIT_SSH_COPY_PUBLIC_KEY)

An error occurred when trying to copy the newly generated
public key of the C<rsync> account to the default SSH dir
of the I<tfsupport> account.

=item Exit status 27 ($EXIT_SSH_ID_FILE)

The path to the SSH identity file for a specified account
could not be found.

=item Exit status 29 ($EXIT_CLOUD_ACCOUNT_NAME)

An error occurred forming the name of the cloud account.

=item Exit status 30 ($EXIT_GENERATE_PERMS)

An error occurred generating the perm file for a backup type.

=item Exit status 31 ($EXIT_UPLOAD_PERMS)

An error occurred uploading a perm file to the cloud server.

=item Exit status 32 ($EXIT_RESTORE_PERMS)

When restoring perms from a perm file via the
B<--restore-from-permfiles> command line option,
there was an error restoring the perms on the files
corresponding to the specified backup type.

=item Exit status 33 ($EXIT_DOWNLOAD_PERMS)

When downloading a perm file via the
B<--download-permfiles> command line option,
the perm file corresponding to the specified backup type
can not be downloaded from the cloud server.

=item Exit status 34 ($EXIT_PERM_FILE_MISSING)

When restoring permissions from a perm file via the
B<--restore-from-permfiles> command line option,
the perm file corresponding to the specified backup type
can not be found.

=item Exit status 40 ($EXIT_LOCK_ACQUISITION)

Could not acquire the appropriate lockfile which means some
previous instance of the script is still running.

=item Exit status 41 ($EXIT_BACKUP_DEVICE_CONFLICT)

If a backup device is specified with B<--device> or
a usb device is specified with B<--usb-device> and
B<--rsync-server> is specified, this error results.
You may not back up to both a device and an rsync server
at the same time.

Likewise, if a backup device is specified with B<--device> or
a usb device is specified with B<--usb-device> and
B<--rsync-dir> is specified and not B<--rsync-server>, this error results.
You may not back up to both a device and a file system
at the same time.

=item Exit status 42 ($EXIT_PLATFORM)

Unknown operating system.

=item Exit status 43 ($EXIT_RSYNC_ERROR)

The F<rsync> command returned an error.
This script response is to stop backup transactions and exit.

=item Exit status 44 ($EXIT_USERS_INFO_SAVE)

The users info files could not be generated and saved when
attempting a backup.

=item Exit status 45 ($EXIT_PSERVER_INFO_SAVE)

The pserver info file could not be saved when
attempting a backup.

=item Exit status 46 ($EXIT_PSERVER_CLOISTER_FILES_SAVE)

The pserver cloister files cound not saved when
attempting a backup.

=item Exit status 50 ($EXIT_XFERDIR_WRITE_ERROR)

An error occurred when attempting to write a file in the xfer dir.

=item Exit status 51 ($EXIT_XFERDIR_MKDIR)

An error occurred when attempting to make the transfer directory.
This could happen during installation on the primary or the
backup server.

=item Exit status 52 ($EXIT_XFERDIR_RMDIR)

An error occurred when attempting to remove the transfer directory.

=item Exit status 53 ($EXIT_INFODIR_MKDIR)

An error occurred when attempting to make the production server info directory.

=item Exit status 54 ($EXIT_INFODIR_RMDIR)

An error occurred when attempting to remove the production server info directory.

=item Exit status 55 ($EXIT_USERSDIR_MKDIR)

An error occurred when attempting to make the users info directory.

=item Exit status 56 ($EXIT_USERSDIR_RMDIR)

An error occurred when attempting to remove the users info directory.

=item Exit status 57 ($EXIT_TOP_LEVEL_MKDIR)

Could not make the top level backup dir, ie
F</usr2/tfrsync> on RTI, or F</d/tfrsync> on Daisy.

=item Exit status 58 ($EXIT_INSTALL_PSERVER_INFO_FILE)

An error occurred when attempting to copy the
info file from the production server to the backup server.

=item Exit status 59 ($EXIT_DOVE_SERVER_MISSING)

During installation on the backup server, the script
verifies that the Dove server script exists and
if it does not, the script exits with this exit status.

=item Exit status 60 ($EXIT_DOVE_SERVER_SAVE_EXISTS)

During installation on the backup server,
the script verifies that the saved Dover server script
does NOT exist, and if it does, the script exits with
this exit status.

=item Exit status 62 ($EXIT_CLOISTERDIR_MKDIR)

An error occurred when attempting to make
the directory for the production server cloistered files.

=item Exit status 63 ($EXIT_CLOISTERDIR_RMDIR)

An error occurred when attempting to remove
the directory for the production server cloistered files.

=item Exit status 64 ($EXIT_RUNTIME_CLEANUP)

Could not cleanup the process lock or the SSH tunnel socket.

=item Exit status 70 ($EXIT_COULD_NOT_EXECUTE)

The script was attempting to exec a program,
typically via the builtin fuction C<system>, and
the exec of the program failed.

=item Exit status 71 ($EXIT_FROM_SIGNAL)

The script execed a program and the program exited
because it caught a signal.

=item Exit status 72 ($EXIT_LOGFILE_SETUP)

An error occurred while attempting to establish the
location for the log files.

=item Exit status 73 ($EXIT_NET_IPADDR)

Could not get the ip address of the network device.

=item Exit status 74 ($EXIT_NET_NETMASK)

Could not get the netmask of the network device.

=item Exit status 75 ($EXIT_NET_GATEWAY)

Could not get the ip address of the gateway (router).

=back


=head1 SEE ALSO

I<rsync(1)>, I<rtibackup.pl>, I<harden_linux.pl>


=cut
