Switched ppss over to md5 checksumming. (backup)
This commit is contained in:
Louwrentius
parent
825d7ed0a1
commit
61d8a24ed2
272
ppss
272
ppss
|
|
@ -25,7 +25,7 @@ trap 'kill_process' SIGINT
|
|||
|
||||
# Setting some vars.
|
||||
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
||||
SCRIPT_VERSION="2.63"
|
||||
SCRIPT_VERSION="2.65"
|
||||
|
||||
# The first argument to this script can be a mode.
|
||||
MODES="node start config stop pause continue deploy status erase kill ec2"
|
||||
|
|
@ -119,6 +119,20 @@ ITEM_ESCAPED=""
|
|||
NODE_STATUS="$PPSS_DIR/status.txt"
|
||||
DAEMON=0
|
||||
|
||||
case $ARCH in
|
||||
"Darwin") MD5=md5 ;;
|
||||
"FreeBSD") MD5=md5 ;;
|
||||
"SunOS") MD5="diget -a md5" ;;
|
||||
"Linux") MD5=md5sum ;;
|
||||
esac
|
||||
|
||||
if [ "$ARCH" == "Darwin" ] || [ "$ARCH" == "FreeBSD" ]
|
||||
then
|
||||
MD5=md5
|
||||
else
|
||||
MD5=$MD5
|
||||
fi
|
||||
|
||||
showusage_short () {
|
||||
|
||||
echo
|
||||
|
|
@ -778,6 +792,12 @@ log () {
|
|||
# Init all vars
|
||||
init_vars () {
|
||||
|
||||
echo "test" | $MD5 > /dev/null 2>&1
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
echo "ERROR - PPSS requires $MD5. It may not be within the path or installed."
|
||||
fi
|
||||
|
||||
create_working_directory
|
||||
|
||||
if [ "$DAEMON" == "1" ]
|
||||
|
|
@ -797,6 +817,13 @@ init_vars () {
|
|||
then
|
||||
rm $LOGFILE
|
||||
fi
|
||||
|
||||
does_file_exist "$PPSS_HOME_DIR"
|
||||
if [ ! "$?" == "0" ] && [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Remote PPSS home directory $PPSS_HOME_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir -p $PPSS_HOME_DIR/$PPSS_DIR"
|
||||
fi
|
||||
|
||||
display_header
|
||||
|
||||
|
|
@ -846,30 +873,38 @@ init_vars () {
|
|||
get_no_of_cpus $HYPERTHREADING
|
||||
fi
|
||||
|
||||
does_file_exist "$PPSS_HOME_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Remote PPSS home directory $PPSS_HOME_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir -p $PPSS_HOME_DIR/$PPSS_DIR"
|
||||
does_file_exist "$PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Remote Job log directory $JOB_lOG_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir $PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
fi
|
||||
elif [ ! -e "$JOB_LOG_DIR" ]
|
||||
then
|
||||
mkdir -p "$JOB_LOG_DIR"
|
||||
fi
|
||||
|
||||
does_file_exist "$PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
if [ -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Remote Job log directory $JOB_lOG_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir $PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
ITEM_LOCK_DIR="$PPSS_DIR/ITEM_LOCK_DIR"
|
||||
fi
|
||||
|
||||
does_file_exist "$ITEM_LOCK_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Creating remote item lock dir."
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Creating remote item lock dir."
|
||||
else
|
||||
log DEBUG "Creating local item lock dir."
|
||||
fi
|
||||
exec_cmd "mkdir $ITEM_LOCK_DIR"
|
||||
fi
|
||||
|
||||
if [ ! -e "$JOB_LOG_DIR" ]
|
||||
then
|
||||
mkdir -p "$JOB_LOG_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Failed to create item lock dir."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
|
|
@ -1415,35 +1450,18 @@ upload_item () {
|
|||
|
||||
lock_item () {
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
ITEM="$1"
|
||||
LOCK_FILE_NAME=`echo "$ITEM" | $MD5 | awk '{ print $1 }'`
|
||||
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
|
||||
log DEBUG "Trying to lock item $ITEM - $ITEM_LOCK_FILE."
|
||||
exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
|
||||
ERROR="$?"
|
||||
|
||||
if [ "$ERROR" == "$?" ]
|
||||
then
|
||||
ITEM="$1"
|
||||
|
||||
LOCK_FILE_NAME=`echo "$ITEM" | \
|
||||
sed s/^\\\.//g | \
|
||||
sed s/^\\\.\\\.//g | \
|
||||
sed s/^\\\///g | \
|
||||
sed s/\\\//\\\\\\ /g | \
|
||||
sed s/\\ /\\\\\\\\\\\\\\ /g | \
|
||||
sed s/\\'/\\\\\\\\\\\\\\'/g | \
|
||||
sed s/\\\//\\\\\\\\\\\\\\ /g | \
|
||||
sed s/\&/\\\\\\\\\\\\\\&/g | \
|
||||
sed s/\;/\\\\\\\\\\\\\\;/g | \
|
||||
sed s/\(/\\\\\\\\\\(/g | \
|
||||
sed s/\)/\\\\\\\\\\)/g `
|
||||
|
||||
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
|
||||
log DEBUG "Trying to lock item $ITEM - $ITEM_LOCK_FILE."
|
||||
exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
|
||||
ERROR="$?"
|
||||
|
||||
if [ "$ERROR" == "$?" ]
|
||||
then
|
||||
exec_cmd "touch $ITEM_LOCK_FILE/$HOSTNAME" # Record that item is claimed by node x.
|
||||
fi
|
||||
|
||||
return "$ERROR"
|
||||
exec_cmd "touch $ITEM_LOCK_FILE/$HOSTNAME" # Record that item is claimed by node x.
|
||||
fi
|
||||
return "$ERROR"
|
||||
}
|
||||
|
||||
get_input_lock () {
|
||||
|
|
@ -1800,7 +1818,8 @@ commando () {
|
|||
#
|
||||
# Create the log file containing the output of the command.
|
||||
#
|
||||
LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g | sed s/\\ /_/g`
|
||||
#LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g | sed s/\\ /_/g`
|
||||
LOG_FILE_NAME=`echo "$ITEM" | $MD5 | awk '{ print $1 }'`
|
||||
ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
|
||||
|
||||
#
|
||||
|
|
@ -1826,104 +1845,97 @@ commando () {
|
|||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
|
||||
does_file_exist "$ITEM_LOG_FILE"
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Skipping item $ITEM - already processed."
|
||||
ERROR=""
|
||||
#
|
||||
# Some formatting of item log files.
|
||||
#
|
||||
DATE=`date +%b\ %d\ %H:%M:%S`
|
||||
echo "===== PPSS Item Log File =====" > "$ITEM_LOG_FILE"
|
||||
echo -e "Host:\t\t$HOSTNAME" >> "$ITEM_LOG_FILE"
|
||||
echo -e "Process:\t$PID" >> "$ITEM_LOG_FILE"
|
||||
echo -e "Item:\t\t$ITEM" >> "$ITEM_LOG_FILE"
|
||||
echo -e "Start date:\t$DATE" >> "$ITEM_LOG_FILE"
|
||||
echo -e "" >> "$ITEM_LOG_FILE"
|
||||
|
||||
#
|
||||
# The actual execution of the command as specified by
|
||||
# the -c option.
|
||||
#
|
||||
BEFORE=`get_time_in_seconds`
|
||||
TMP=`echo $COMMAND | grep -i '$ITEM'`
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1
|
||||
ERROR="$?"
|
||||
MYPID="$!"
|
||||
else
|
||||
ERROR=""
|
||||
#
|
||||
# Some formatting of item log files.
|
||||
#
|
||||
DATE=`date +%b\ %d\ %H:%M:%S`
|
||||
echo "===== PPSS Item Log File =====" > "$ITEM_LOG_FILE"
|
||||
echo -e "Host:\t\t$HOSTNAME" >> "$ITEM_LOG_FILE"
|
||||
echo -e "Process:\t$PID" >> "$ITEM_LOG_FILE"
|
||||
echo -e "Item:\t\t$ITEM" >> "$ITEM_LOG_FILE"
|
||||
echo -e "Start date:\t$DATE" >> "$ITEM_LOG_FILE"
|
||||
echo -e "" >> "$ITEM_LOG_FILE"
|
||||
|
||||
#
|
||||
# The actual execution of the command as specified by
|
||||
# the -c option.
|
||||
#
|
||||
BEFORE=`get_time_in_seconds`
|
||||
TMP=`echo $COMMAND | grep -i '$ITEM'`
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1
|
||||
ERROR="$?"
|
||||
MYPID="$!"
|
||||
else
|
||||
eval '$COMMAND"$ITEM" >> "$ITEM_LOG_FILE" 2>&1'
|
||||
ERROR="$?"
|
||||
MYPID="$!"
|
||||
fi
|
||||
AFTER=`get_time_in_seconds`
|
||||
eval '$COMMAND"$ITEM" >> "$ITEM_LOG_FILE" 2>&1'
|
||||
ERROR="$?"
|
||||
MYPID="$!"
|
||||
fi
|
||||
AFTER=`get_time_in_seconds`
|
||||
|
||||
echo -e "" >> "$ITEM_LOG_FILE"
|
||||
echo -e "" >> "$ITEM_LOG_FILE"
|
||||
|
||||
# Some error logging. Success or fail.
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
# Some error logging. Success or fail.
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
echo -e "Status:\t\tFAILURE" >> "$ITEM_LOG_FILE"
|
||||
else
|
||||
echo -e "Status:\t\tSUCCESS" >> "$ITEM_LOG_FILE"
|
||||
fi
|
||||
|
||||
#
|
||||
# If part of a cluster, remove the downloaded item after
|
||||
# it has been processed and uploaded as not to fill up disk space.
|
||||
#
|
||||
if [ "$DOWNLOAD_TO_NODE" == "1" ]
|
||||
then
|
||||
if [ -e "$ITEM" ]
|
||||
then
|
||||
echo -e "Status:\t\tFAILURE" >> "$ITEM_LOG_FILE"
|
||||
else
|
||||
echo -e "Status:\t\tSUCCESS" >> "$ITEM_LOG_FILE"
|
||||
rm -rf "$ITEM"
|
||||
else
|
||||
log DEBUG "There is no local file to remove.. strange..."
|
||||
fi
|
||||
|
||||
#
|
||||
# If part of a cluster, remove the downloaded item after
|
||||
# it has been processed and uploaded as not to fill up disk space.
|
||||
#
|
||||
if [ "$DOWNLOAD_TO_NODE" == "1" ]
|
||||
then
|
||||
if [ -e "$ITEM" ]
|
||||
then
|
||||
rm -rf "$ITEM"
|
||||
else
|
||||
log DEBUG "There is no local file to remove.. strange..."
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Create remote output dir and transfer output to server.
|
||||
#
|
||||
escape_item "$DIR_NAME"
|
||||
ITEM_OUTPUT_DIR="$REMOTE_OUTPUT_DIR/$ITEM_ESCAPED"
|
||||
|
||||
if [ "$DOWNLOAD_TO_NODE" == "0" ]
|
||||
then
|
||||
log DEBUG "Download to node is disabled."
|
||||
else
|
||||
if [ "$DIR_NAME" == "." ]
|
||||
then
|
||||
DIR_NAME=""
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Create remote output dir and transfer output to server.
|
||||
#
|
||||
escape_item "$DIR_NAME"
|
||||
ITEM_OUTPUT_DIR="$REMOTE_OUTPUT_DIR/$ITEM_ESCAPED"
|
||||
#
|
||||
# Upload the output file back to the server.
|
||||
#
|
||||
|
||||
if [ "$DOWNLOAD_TO_NODE" == "0" ]
|
||||
upload_item "$OUTPUT_DIR" "$DIR_NAME"
|
||||
|
||||
#
|
||||
# Upload the log file to the server.
|
||||
#
|
||||
|
||||
elapsed "$BEFORE" "$AFTER" >> "$ITEM_LOG_FILE"
|
||||
echo -e "" >> "$ITEM_LOG_FILE"
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Uploading item log file $ITEM_LOG_FILE to master $PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
scp -q $SSH_OPTS $SSH_KEY "$ITEM_LOG_FILE" $USER@$SSH_SERVER:$PPSS_HOME_DIR/$JOB_LOG_DIR
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Download to node is disabled."
|
||||
else
|
||||
if [ "$DIR_NAME" == "." ]
|
||||
then
|
||||
DIR_NAME=""
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Upload the output file back to the server.
|
||||
#
|
||||
|
||||
upload_item "$OUTPUT_DIR" "$DIR_NAME"
|
||||
|
||||
#
|
||||
# Upload the log file to the server.
|
||||
#
|
||||
|
||||
elapsed "$BEFORE" "$AFTER" >> "$ITEM_LOG_FILE"
|
||||
echo -e "" >> "$ITEM_LOG_FILE"
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Uploading item log file $ITEM_LOG_FILE to master $PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
scp -q $SSH_OPTS $SSH_KEY "$ITEM_LOG_FILE" $USER@$SSH_SERVER:$PPSS_HOME_DIR/$JOB_LOG_DIR
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Uploading of item log file failed."
|
||||
fi
|
||||
log DEBUG "Uploading of item log file failed."
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
|
|||
10
ppss-test.sh
10
ppss-test.sh
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
DEBUG="$1"
|
||||
VERSION="2.63"
|
||||
VERSION="2.65"
|
||||
TMP_DIR="ppss"
|
||||
PPSS=./ppss
|
||||
PPSS_DIR=ppss_dir
|
||||
|
|
@ -170,8 +170,8 @@ testSkippingOfProcessedItems () {
|
|||
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||
assertNull "PPSS retured some errors..." "$RES"
|
||||
|
||||
grep -i skip ./$PPSS_DIR/* >> /dev/null 2>&1
|
||||
assertEquals "Skipping of items went wrong." 0 "$?"
|
||||
RES=`grep -c -i locked ./$PPSS_DIR/ppss-log* | tail -n 1 | cut -d ":" -f 2`
|
||||
assertEquals "Skipping of items went wrong." 2 "$RES"
|
||||
|
||||
rename-ppss-dir $FUNCNAME-1
|
||||
|
||||
|
|
@ -183,8 +183,8 @@ testSkippingOfProcessedItems () {
|
|||
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||
assertNull "PPSS retured some errors..." "$RES"
|
||||
|
||||
grep -i skip ./$PPSS_DIR/* >> /dev/null 2>&1
|
||||
assertEquals "Skipping of items went wrong." 0 "$?"
|
||||
RES=`grep -c -i locked ./$PPSS_DIR/ppss-log* | tail -n 1 | cut -d ":" -f 2`
|
||||
assertEquals "Skipping of items went wrong." 8 "$RES"
|
||||
|
||||
rm -rf "/tmp/$TMP_DIR"
|
||||
rename-ppss-dir $FUNCNAME-2
|
||||
|
|
|
|||
Loading…
Reference in New Issue