Released version 2.56b2

This commit is contained in:
Louwrentius 2010-02-09 22:01:05 +00:00
parent 9790e52c89
commit 365705e8c7
2 changed files with 126 additions and 75 deletions

148
ppss
View File

@ -25,7 +25,7 @@ trap 'kill_process' SIGINT
# Setting some vars.
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.56"
SCRIPT_VERSION="2.56b2"
# The first argument to this script can be a mode.
MODES="node start config stop pause continue deploy status erase kill"
@ -67,6 +67,7 @@ CPUINFO=/proc/cpuinfo
PROCESSORS=""
STOP_KEY=$RANDOM$RANDOM$RANDOM
KILL_KEY=$RANDOM$RANDOM$RANDOM
RECURSION="1"
SSH_SERVER="" # Remote server or 'master'.
SSH_KEY="" # SSH key for ssh account.
@ -97,8 +98,8 @@ showusage_short () {
echo "|P|P|S|S| $SCRIPT_NAME $SCRIPT_VERSION"
echo
echo "usage: $0 [ -d <sourcedir> | -f <sourcefile> ] [ -c '<command> \"\$ITEM\"' ]"
echo " [ -C <configfile> ] [ -j ] [ -l <logfile> ] [ -p <# jobs> ]"
echo " [ -D <delay> ] [ -h ] [ --help ]"
echo " [ -C <configfile> ] [ -j ] [ -l <logfile> ] [ -p <# jobs> ]"
echo " [ -D <delay> ] [ -h ] [ --help ] [ -r ] "
echo
echo "Examples:"
echo " $0 -d /dir/with/some/files -c 'gzip '"
@ -143,6 +144,10 @@ showusage_normal () {
echo
echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread"
echo -e " the load. The delay is only used at the start of all 'threads'."
echo
echo -e "--no-recurse | -r By default, recursion of directories is enabled when the -d option is "
echo -e " used. If this is not prefered, this can be disabled with this option "
echo -e " Only files within the specified directory will be processed."
echo
echo -e "Example: encoding some wav files to mp3 using lame:"
echo
@ -206,6 +211,10 @@ showusage_long () {
echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread"
echo -e " the load. The delay is only used at the start of all 'threads'."
echo
echo -e "--no-recurse | -r By default, recursion of directories is enabled when the -d option is "
echo -e " used. If this is not prefered, this can be disabled with this option "
echo -e " Only files within the specified directory will be processed."
echo
echo -e "The following options are used for distributed execution of PPSS."
echo
echo -e "--master | -m Specifies the SSH server that is used for communication between nodes."
@ -463,6 +472,11 @@ do
add_var_to_config LOGFILE "$LOGFILE"
shift 2
;;
--no-recurse|-r )
RECURSION="0"
add_var_to_config LOGFILE "$RECURSION"
shift 1
;;
--workingdir|-w )
WORKINGDIR="$2"
add_var_to_config WORKINGDIR "$WORKINGDIR"
@ -542,13 +556,13 @@ done
display_header () {
log info ""
log INFO "========================================================="
log INFO " |P|P|S|S| "
log INFO "$SCRIPT_NAME version $SCRIPT_VERSION"
log INFO "========================================================="
log INFO "Hostname:\t\t$HOSTNAME"
log INFO "---------------------------------------------------------"
log DSPLY ""
log DSPLY "========================================================="
log DSPLY " |P|P|S|S| "
log DSPLY "$SCRIPT_NAME vers. $SCRIPT_VERSION"
log DSPLY "========================================================="
log DSPLY "Hostname:\t\t$HOSTNAME"
log DSPLY "---------------------------------------------------------"
}
create_working_directory () {
@ -605,20 +619,20 @@ init_vars () {
if [ -e "$CPUINFO" ]
then
CPU=`cat /proc/cpuinfo | grep 'model name' | cut -d ":" -f 2 | sed -e s/^\ //g | sort | uniq`
log INFO "CPU: $CPU"
log DSPLY "CPU: $CPU"
elif [ "$ARCH" == "Darwin" ]
then
MODEL=`system_profiler SPHardwareDataType | grep "Processor Name" | cut -d ":" -f 2`
SPEED=`system_profiler SPHardwareDataType | grep "Processor Speed" | cut -d ":" -f 2`
log INFO "CPU: $MODEL $SPEED"
log DSPLY "CPU: $MODEL $SPEED"
elif [ "$ARCH" == "SunOS" ]
then
CPU=`psrinfo -v | grep MHz | cut -d " " -f 4,8 | awk '{ printf ("Processor architecture: %s @ %s MHz.\n", $1,$2) }' | head -n 1`
log INFO "$CPU"
log DSPLY "$CPU"
else
log INFO "CPU: Cannot determine. Provide a patch for your arch!"
log INFO "Arch is $ARCH"
log DSPLY "CPU: Cannot determine. Provide a patch for your arch!"
log DSPLY "Arch is $ARCH"
fi
if [ -z "$MAX_NO_OF_RUNNING_JOBS" ]
@ -712,9 +726,16 @@ log () {
LOG_MSG="$PREFIX $MESG"
ECHO_MSG="$PREFIX_SMALL $MESG"
echo -e "$LOG_MSG" >> "$LOGFILE"
if [ ! -z "$PPSSDEBUG" ] && [ ! "$PPSSDEBUG" == "0" ]
then
echo -e "$LOG_MSG" >> "$LOGFILE"
if [ "$TYPE" == "INFO" ] || [ "$TYPE" == "ERROR" ] || [ "$TYPE" == "WARN" ]
elif [ "$TYPE" == "INFO" ] || [ "$TYPE" == "ERROR" ] || [ "$TYPE" == "WARN" ] || [ "$TYPE" == "DSPLY" ]
then
echo -e "$LOG_MSG" >> "$LOGFILE"
fi
if [ "$TYPE" == "DSPLY" ] || [ "$TYPE" == "ERROR" ] || [ "$TYPE" == "WARN" ]
then
echo -e "$ECHO_MSG"
fi
@ -751,7 +772,7 @@ erase_ppss () {
ssh -q $SSH_KEY $SSH_OPTS $USER@$NODE "rm -rf $PPSS_HOME_DIR"
done
else
log INFO "Aborting.."
log DSPLY "Aborting.."
fi
sleep 1
}
@ -813,9 +834,9 @@ deploy () {
if [ "$ERROR" == "0" ]
then
log INFO "PPSS installed on node $NODE."
log DSPLY "PPSS installed on node $NODE."
else
log INFO "PPSS failed to install on $NODE."
log DSPLY "PPSS failed to install on $NODE."
fi
kill $SSH_PID
@ -826,7 +847,7 @@ deploy_ppss () {
if [ -z "$NODES_FILE" ]
then
log INFO "ERROR - are you using the right option? -C ?"
log ERROR "ERROR - are you using the right option? -C ?"
set_status ERROR
cleanup
exit 1
@ -882,7 +903,7 @@ deploy_ppss () {
start_ppss_on_node () {
NODE="$1"
log INFO "Starting PPSS on node $NODE."
log DSPLY "Starting PPSS on node $NODE."
ssh $SSH_KEY $USER@$NODE -o ConnectTimeout=5 "cd $PPSS_HOME_DIR ; screen -d -m -S PPSS ~/$PPSS_HOME_DIR/$0 node --config ~/$PPSS_HOME_DIR/$CONFIG"
}
@ -951,12 +972,12 @@ get_no_of_cpus () {
if [ ! -z "$NUMBER" ]
then
log INFO "Found $NUMBER logic processors."
log DSPLY "Found $NUMBER logic processors."
fi
elif [ "$HPT" == "no" ]
then
log INFO "Hyperthreading is disabled."
log DSPLY "Hyperthreading is disabled."
if [ "$ARCH" == "Linux" ]
then
@ -966,9 +987,9 @@ get_no_of_cpus () {
PHYSICAL=`grep 'physical id' $CPUINFO | sort | uniq | wc -l`
if [ "$PHYSICAL" == "1" ]
then
log INFO "Found $PHYSICAL physical CPU."
log DSPLY "Found $PHYSICAL physical CPU."
else
log INFO "Found $PHYSICAL physical CPUs."
log DSPLY "Found $PHYSICAL physical CPUs."
fi
TMP=`grep 'core id' $CPUINFO`
@ -976,10 +997,10 @@ get_no_of_cpus () {
then
log DEBUG "Starting job only for each physical core on all physical CPU(s)."
NUMBER=`grep 'core id' $CPUINFO | sort | uniq | wc -l`
log INFO "Found $NUMBER physical cores."
log DSPLY "Found $NUMBER physical cores."
else
log INFO "Single core processor(s) detected."
log INFO "Starting job for each physical CPU."
log DSPLY "Single core processor(s) detected."
log DSPLY "Starting job for each physical CPU."
NUMBER=$PHYSICAL
fi
else
@ -1200,12 +1221,25 @@ get_all_items () {
then
if [ ! -z "$SSH_SERVER" ] # Are we running stand-alone or as a slave?"
then
ITEMS=`exec_cmd "find $SRC_DIR -type f"`
check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
if [ "$RECURSION" == "1" ]
then
ITEMS=`exec_cmd "find $SRC_DIR ! -type d"`
check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
else
log DEBUG "Recursion is disabled."
ITEMS=`exec_cmd "ls -1 $SRC_DIR"`
check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
fi
else
if [ -e "$SRC_DIR" ]
then
ITEMS=`find $SRC_DIR -type f`
if [ "$RECURSION" == "1" ]
then
ITEMS=`find "$SRC_DIR" ! -type d`
else
log DEBUG "Recursion is disabled."
ITEMS=`ls -1 "$SRC_DIR"`
fi
else
ITEMS=""
fi
@ -1537,9 +1571,9 @@ listen_for_job () {
RES=$((MAX_NO_OF_RUNNING_JOBS-DIED))
if [ "$RES" == "1" ]
then
log INFO "$((MAX_NO_OF_RUNNING_JOBS-DIED)) job is remaining. "
log DSPLY "$((MAX_NO_OF_RUNNING_JOBS-DIED)) job is remaining. "
else
log INFO "$((MAX_NO_OF_RUNNING_JOBS-DIED)) jobs are remaining."
log DSPLY "$((MAX_NO_OF_RUNNING_JOBS-DIED)) jobs are remaining."
echo -en "\033[1A"
fi
fi
@ -1596,7 +1630,7 @@ listen_for_job () {
PERCENT=$((100 * $ARRAY_POINTER / $SIZE_OF_ARRAY ))
if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
then
log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."
log DSPLY "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."
if [ "$PERCENT" == "100" ]
then
FINISHED=1
@ -1611,10 +1645,10 @@ listen_for_job () {
if [ ! "$PERCENT" == "100" ]
then
echo
log INFO "Finished. Consult $JOB_LOG_DIR for job output."
log INFO "Press ENTER to continue."
log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
log DSPLY "Press ENTER to continue."
else
log INFO "Finished. Consult $JOB_LOG_DIR for job output."
log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
fi
cleanup
}
@ -1624,11 +1658,11 @@ start_all_workers () {
if [ "$MAX_NO_OF_RUNNING_JOBS" == "1" ]
then
log INFO "Starting $MAX_NO_OF_RUNNING_JOBS single worker."
log DSPLY "Starting $MAX_NO_OF_RUNNING_JOBS single worker."
else
log INFO "Starting $MAX_NO_OF_RUNNING_JOBS parallel workers."
log DSPLY "Starting $MAX_NO_OF_RUNNING_JOBS parallel workers."
fi
log INFO "---------------------------------------------------------"
log DSPLY "---------------------------------------------------------"
i=0
while [ "$i" -lt "$MAX_NO_OF_RUNNING_JOBS" ]
@ -1673,20 +1707,20 @@ show_status () {
PROCESSED=`exec_cmd "ls -1 $ITEM_LOCK_DIR | wc -l"` 2>&1 >> /dev/null
TMP_STATUS=$((100 * $PROCESSED / $ITEMS))
log INFO "Status:\t\t$TMP_STATUS percent complete."
log DSPLY "Status:\t\t$TMP_STATUS percent complete."
if [ ! -z $NODES_FILE ]
then
TMP_NO=`cat $NODES_FILE | wc -l`
log INFO "Nodes:\t $TMP_NO"
log DSPLY "Nodes:\t $TMP_NO"
fi
log INFO "Items:\t\t$ITEMS"
log DSPLY "Items:\t\t$ITEMS"
log INFO "---------------------------------------------------------"
log DSPLY "---------------------------------------------------------"
HEADER=`echo IP-address Hostname Processed Status | awk '{ printf ("%-16s %-18s % 10s %10s\n",$1,$2,$3,$4) }'`
log INFO "$HEADER"
log INFO "---------------------------------------------------------"
log DSPLY "$HEADER"
log DSPLY "---------------------------------------------------------"
PROCESSED=0
for x in `cat $NODES_FILE`
do
@ -1705,11 +1739,11 @@ show_status () {
fi
let PROCESSED=$PROCESSED+$RES
LINE=`echo "$x $NODE $RES $STATUS" | awk '{ printf ("%-16s %-18s % 10s %10s\n",$1,$2,$3,$4) }'`
log INFO "$LINE"
log DSPLY "$LINE"
done
log INFO "---------------------------------------------------------"
log DSPLY "---------------------------------------------------------"
LINE=`echo $PROCESSED | awk '{ printf ("Total processed: % 29s\n",$1) }'`
log INFO "$LINE"
log DSPLY "$LINE"
}
@ -1747,7 +1781,7 @@ main () {
config )
LOGFILE=/dev/null
display_header
log INFO "Generating configuration file $CONFIG"
log DSPLY "Generating configuration file $CONFIG"
add_var_to_config PPSS_LOCAL_TMPDIR "$PPSS_LOCAL_TMPDIR"
add_var_to_config PPSS_LOCAL_OUTPUT "$PPSS_LOCAL_OUTPUT"
cleanup
@ -1757,7 +1791,7 @@ main () {
stop )
LOGFILE=/dev/null
display_header
log INFO "Stopping PPSS on all nodes."
log DSPLY "Stopping PPSS on all nodes."
exec_cmd "touch $STOP_SIGNAL"
cleanup
exit
@ -1765,7 +1799,7 @@ main () {
pause )
LOGFILE=/dev/null
display_header
log INFO "Pausing PPSS on all nodes."
log DSPLY "Pausing PPSS on all nodes."
exec_cmd "touch $PAUSE_SIGNAL"
cleanup
exit
@ -1775,12 +1809,12 @@ main () {
display_header
if does_file_exist "$STOP_SIGNAL"
then
log INFO "Continuing processing, please use $0 start to start PPSS on al nodes."
log DSPLY "Continuing processing, please use $0 start to start PPSS on al nodes."
exec_cmd "rm -f $STOP_SIGNAL"
fi
if does_file_exist "$PAUSE_SIGNAL"
then
log INFO "Continuing PPSS on all nodes."
log DSPLY "Continuing PPSS on all nodes."
exec_cmd "rm -f $PAUSE_SIGNAL"
fi
cleanup
@ -1789,7 +1823,7 @@ main () {
deploy )
LOGFILE=/dev/null
display_header
log INFO "Deploying PPSS on nodes."
log DSPLY "Deploying PPSS on nodes."
deploy_ppss
wait
cleanup
@ -1805,7 +1839,7 @@ main () {
erase )
LOGFILE=/dev/null
display_header
log INFO "Erasing PPSS from all nodes."
log DSPLY "Erasing PPSS from all nodes."
erase_ppss
cleanup
exit 0

View File

@ -1,7 +1,7 @@
#!/bin/bash
DEBUG="$1"
VERSION=2.56
VERSION="2.56b2"
TMP_DIR="ppss"
PPSS=./ppss
PPSS_DIR=ppss_dir
@ -37,16 +37,6 @@ oneTimeSetUp () {
REMOVEFILES="$PPSS_DIR test-ppss-*"
cleanup
for x in $NORMALTESTFILES
do
echo "$x" >> "$INPUTFILENORMAL"
done
for x in $SPECIALTESTFILES
do
echo $x >> "$INPUTFILESPECIAL"
done
}
testVersion () {
@ -85,19 +75,42 @@ createDirectoryWithSomeFiles () {
A="File with Spaces"
B="File\With\Slashes"
c="symnlink1"
d="symnlink2"
TMP_FILE="/tmp/$TMP_DIR"
if [ ! -e "$TMP_FILE" ]
then
mkdir "$TMP_FILE"
fi
touch "$A"
touch "$B"
ln -s /etc/resolve.conf "$TMP_FILE"/
ln -s /etc/hosts "$TMP_FILE"/
}
testRecursion () {
createDirectoryWithSomeFiles
#Execution of PPSS with recursion disabled.
RES=$( { ./$PPSS -d /tmp/$TMP_DIR -c 'ls -alh ' -r >> /dev/null ; } 2>&1 )
assertEquals "PPSS did not execute properly." 0 "$?"
NUMBER=`find /tmp/$TMP_DIR ! -type d | wc -l`
LOGS=`ls -1 $JOBLOG/* | wc -l`
assertEquals "Did not find equal files and joblogs $TMP_FILE" "$NUMBER" "$LOGS"
rm -rf "/tmp/$TMP_DIR"
rename-ppss-dir $FUNCNAME
mkdir "/tmp/$TMP_DIR"
for x in "$A" "$B"
do
TMP_FILE="/tmp/$TMP_DIR/$x"
touch "$TMP_FILE"
done
}
testSpacesInFilenames () {
createDirectoryWithSomeFiles
#Regular execution of PPSS
RES=$( { ./$PPSS -d /tmp/$TMP_DIR -c 'ls -alh ' >> /dev/null ; } 2>&1 )
assertEquals "PPSS did not execute properly." 0 "$?"
@ -110,6 +123,10 @@ testSpacesInFilenames () {
grep "SUCCESS" $JOBLOG/* >> /dev/null 2>&1
assertEquals "Found error with space in filename $TMP_FILE" "0" "$?"
NUMBER=`find /tmp/$TMP_DIR ! -type d | wc -l`
LOGS=`ls -1 $JOBLOG/* | wc -l`
assertEquals "Did not find equal files and joblogs $TMP_FILE" "$NUMBER" "$LOGS"
rm -rf "/tmp/$TMP_DIR"
rename-ppss-dir $FUNCNAME
}