Switched ppss over to md5 checksumming. (backup)
This commit is contained in:
Louwrentius
parent
825d7ed0a1
commit
61d8a24ed2
90
ppss
90
ppss
|
|
@ -25,7 +25,7 @@ trap 'kill_process' SIGINT
|
|||
|
||||
# Setting some vars.
|
||||
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
||||
SCRIPT_VERSION="2.63"
|
||||
SCRIPT_VERSION="2.65"
|
||||
|
||||
# The first argument to this script can be a mode.
|
||||
MODES="node start config stop pause continue deploy status erase kill ec2"
|
||||
|
|
@ -119,6 +119,20 @@ ITEM_ESCAPED=""
|
|||
NODE_STATUS="$PPSS_DIR/status.txt"
|
||||
DAEMON=0
|
||||
|
||||
case $ARCH in
|
||||
"Darwin") MD5=md5 ;;
|
||||
"FreeBSD") MD5=md5 ;;
|
||||
"SunOS") MD5="diget -a md5" ;;
|
||||
"Linux") MD5=md5sum ;;
|
||||
esac
|
||||
|
||||
if [ "$ARCH" == "Darwin" ] || [ "$ARCH" == "FreeBSD" ]
|
||||
then
|
||||
MD5=md5
|
||||
else
|
||||
MD5=$MD5
|
||||
fi
|
||||
|
||||
showusage_short () {
|
||||
|
||||
echo
|
||||
|
|
@ -778,6 +792,12 @@ log () {
|
|||
# Init all vars
|
||||
init_vars () {
|
||||
|
||||
echo "test" | $MD5 > /dev/null 2>&1
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
echo "ERROR - PPSS requires $MD5. It may not be within the path or installed."
|
||||
fi
|
||||
|
||||
create_working_directory
|
||||
|
||||
if [ "$DAEMON" == "1" ]
|
||||
|
|
@ -798,6 +818,13 @@ init_vars () {
|
|||
rm $LOGFILE
|
||||
fi
|
||||
|
||||
does_file_exist "$PPSS_HOME_DIR"
|
||||
if [ ! "$?" == "0" ] && [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Remote PPSS home directory $PPSS_HOME_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir -p $PPSS_HOME_DIR/$PPSS_DIR"
|
||||
fi
|
||||
|
||||
display_header
|
||||
|
||||
if [ -z "$COMMAND" ]
|
||||
|
|
@ -846,30 +873,38 @@ init_vars () {
|
|||
get_no_of_cpus $HYPERTHREADING
|
||||
fi
|
||||
|
||||
does_file_exist "$PPSS_HOME_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
log DEBUG "Remote PPSS home directory $PPSS_HOME_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir -p $PPSS_HOME_DIR/$PPSS_DIR"
|
||||
fi
|
||||
|
||||
does_file_exist "$PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Remote Job log directory $JOB_lOG_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir $PPSS_HOME_DIR/$JOB_LOG_DIR"
|
||||
fi
|
||||
elif [ ! -e "$JOB_LOG_DIR" ]
|
||||
then
|
||||
mkdir -p "$JOB_LOG_DIR"
|
||||
fi
|
||||
|
||||
if [ -z "$SSH_SERVER" ]
|
||||
then
|
||||
ITEM_LOCK_DIR="$PPSS_DIR/ITEM_LOCK_DIR"
|
||||
fi
|
||||
|
||||
does_file_exist "$ITEM_LOCK_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Creating remote item lock dir."
|
||||
exec_cmd "mkdir $ITEM_LOCK_DIR"
|
||||
fi
|
||||
|
||||
if [ ! -e "$JOB_LOG_DIR" ]
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
mkdir -p "$JOB_LOG_DIR"
|
||||
log DEBUG "Creating remote item lock dir."
|
||||
else
|
||||
log DEBUG "Creating local item lock dir."
|
||||
fi
|
||||
exec_cmd "mkdir $ITEM_LOCK_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Failed to create item lock dir."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
|
|
@ -1415,23 +1450,8 @@ upload_item () {
|
|||
|
||||
lock_item () {
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
ITEM="$1"
|
||||
|
||||
LOCK_FILE_NAME=`echo "$ITEM" | \
|
||||
sed s/^\\\.//g | \
|
||||
sed s/^\\\.\\\.//g | \
|
||||
sed s/^\\\///g | \
|
||||
sed s/\\\//\\\\\\ /g | \
|
||||
sed s/\\ /\\\\\\\\\\\\\\ /g | \
|
||||
sed s/\\'/\\\\\\\\\\\\\\'/g | \
|
||||
sed s/\\\//\\\\\\\\\\\\\\ /g | \
|
||||
sed s/\&/\\\\\\\\\\\\\\&/g | \
|
||||
sed s/\;/\\\\\\\\\\\\\\;/g | \
|
||||
sed s/\(/\\\\\\\\\\(/g | \
|
||||
sed s/\)/\\\\\\\\\\)/g `
|
||||
|
||||
LOCK_FILE_NAME=`echo "$ITEM" | $MD5 | awk '{ print $1 }'`
|
||||
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
|
||||
log DEBUG "Trying to lock item $ITEM - $ITEM_LOCK_FILE."
|
||||
exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
|
||||
|
|
@ -1441,9 +1461,7 @@ lock_item () {
|
|||
then
|
||||
exec_cmd "touch $ITEM_LOCK_FILE/$HOSTNAME" # Record that item is claimed by node x.
|
||||
fi
|
||||
|
||||
return "$ERROR"
|
||||
fi
|
||||
}
|
||||
|
||||
get_input_lock () {
|
||||
|
|
@ -1800,7 +1818,8 @@ commando () {
|
|||
#
|
||||
# Create the log file containing the output of the command.
|
||||
#
|
||||
LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g | sed s/\\ /_/g`
|
||||
#LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g | sed s/\\ /_/g`
|
||||
LOG_FILE_NAME=`echo "$ITEM" | $MD5 | awk '{ print $1 }'`
|
||||
ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
|
||||
|
||||
#
|
||||
|
|
@ -1826,12 +1845,6 @@ commando () {
|
|||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
|
||||
does_file_exist "$ITEM_LOG_FILE"
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Skipping item $ITEM - already processed."
|
||||
else
|
||||
ERROR=""
|
||||
#
|
||||
# Some formatting of item log files.
|
||||
|
|
@ -1925,7 +1938,6 @@ commando () {
|
|||
log DEBUG "Uploading of item log file failed."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
start_single_worker
|
||||
return $?
|
||||
|
|
|
|||
10
ppss-test.sh
10
ppss-test.sh
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
DEBUG="$1"
|
||||
VERSION="2.63"
|
||||
VERSION="2.65"
|
||||
TMP_DIR="ppss"
|
||||
PPSS=./ppss
|
||||
PPSS_DIR=ppss_dir
|
||||
|
|
@ -170,8 +170,8 @@ testSkippingOfProcessedItems () {
|
|||
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||
assertNull "PPSS retured some errors..." "$RES"
|
||||
|
||||
grep -i skip ./$PPSS_DIR/* >> /dev/null 2>&1
|
||||
assertEquals "Skipping of items went wrong." 0 "$?"
|
||||
RES=`grep -c -i locked ./$PPSS_DIR/ppss-log* | tail -n 1 | cut -d ":" -f 2`
|
||||
assertEquals "Skipping of items went wrong." 2 "$RES"
|
||||
|
||||
rename-ppss-dir $FUNCNAME-1
|
||||
|
||||
|
|
@ -183,8 +183,8 @@ testSkippingOfProcessedItems () {
|
|||
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||
assertNull "PPSS retured some errors..." "$RES"
|
||||
|
||||
grep -i skip ./$PPSS_DIR/* >> /dev/null 2>&1
|
||||
assertEquals "Skipping of items went wrong." 0 "$?"
|
||||
RES=`grep -c -i locked ./$PPSS_DIR/ppss-log* | tail -n 1 | cut -d ":" -f 2`
|
||||
assertEquals "Skipping of items went wrong." 8 "$RES"
|
||||
|
||||
rm -rf "/tmp/$TMP_DIR"
|
||||
rename-ppss-dir $FUNCNAME-2
|
||||
|
|
|
|||
Loading…
Reference in New Issue