Significant bug fixing and testing. Updated to 2.45
This commit is contained in:
parent
fffccc3e04
commit
20e520e820
|
@ -0,0 +1,146 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
DEBUG="$1"
|
||||||
|
VERSION=2.45
|
||||||
|
|
||||||
|
cleanup () {
|
||||||
|
|
||||||
|
for x in $REMOVEFILES
|
||||||
|
do
|
||||||
|
if [ -e ./$x ]
|
||||||
|
then
|
||||||
|
rm -r ./$x
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
oneTimeSetUp () {
|
||||||
|
|
||||||
|
NORMALTESTFILES=`echo test-{a..z}`
|
||||||
|
SPECIALTESTFILES="\'file-!@#$%^&*()_+=-0987654321~\' \'file-/\<>?:;'{}[]\' file-/\/\:\/!@#$%^&*()_+=-0987654321~ file-/\<>?:;'{}[] http://www.google.nl ftp://storage.nl"
|
||||||
|
JOBLOG=./ppss/job_log
|
||||||
|
INPUTFILENORMAL=test-normal.input
|
||||||
|
INPUTFILESPECIAL=test-special.input
|
||||||
|
LOCALOUTPUT=ppss/PPSS_LOCAL_OUTPUT
|
||||||
|
|
||||||
|
REMOVEFILES="$INPUTFILENORMAL $INPUTFILESPECIAL ppss test-ppss-*"
|
||||||
|
|
||||||
|
cleanup
|
||||||
|
|
||||||
|
for x in $NORMALTESTFILES
|
||||||
|
do
|
||||||
|
echo "$x" >> "$INPUTFILENORMAL"
|
||||||
|
done
|
||||||
|
|
||||||
|
for x in $SPECIALTESTFILES
|
||||||
|
do
|
||||||
|
echo $x >> "$INPUTFILESPECIAL"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
testVersion () {
|
||||||
|
|
||||||
|
RES=`./ppss.sh -v`
|
||||||
|
|
||||||
|
for x in $RES
|
||||||
|
do
|
||||||
|
echo "$x" | grep [0-9] >> /dev/null
|
||||||
|
if [ "$?" == "0" ]
|
||||||
|
then
|
||||||
|
assertEquals "Version mismatch!" "$VERSION" "$x"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
rename-ppss-dir () {
|
||||||
|
|
||||||
|
TEST="$1"
|
||||||
|
|
||||||
|
if [ -e "ppss" ] && [ -d "ppss" ] && [ ! -z "$TEST" ]
|
||||||
|
then
|
||||||
|
mv ppss test-ppss-"$TEST"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
oneTimeTearDown () {
|
||||||
|
|
||||||
|
if [ ! "$DEBUG" == "debug" ]
|
||||||
|
then
|
||||||
|
cleanup
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
testSpecialCharacterHandling () {
|
||||||
|
|
||||||
|
RES=$( { ./ppss.sh -f "$INPUTFILESPECIAL" -c 'echo ' >> /dev/null ; } 2>&1 )
|
||||||
|
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||||
|
|
||||||
|
assertNull "PPSS retured some errors..." "$RES"
|
||||||
|
if [ ! "$?" == "0" ]
|
||||||
|
then
|
||||||
|
echo "RES IS $RES"
|
||||||
|
fi
|
||||||
|
|
||||||
|
RES=`find ppss/PPSS_LOCAL_OUTPUT | wc -l`
|
||||||
|
assertEquals "To many lock files..." "7" "$RES"
|
||||||
|
|
||||||
|
RES1=`ls -1 $JOBLOG`
|
||||||
|
RES2=`ls -1 $LOCALOUTPUT`
|
||||||
|
|
||||||
|
assertEquals "RES1 $RES1 is not the same as RES2 $RES2" "$RES1" "$RES2"
|
||||||
|
|
||||||
|
rename-ppss-dir $FUNCNAME
|
||||||
|
}
|
||||||
|
|
||||||
|
testExistLogFiles () {
|
||||||
|
|
||||||
|
./ppss.sh -f "$INPUTFILENORMAL" -c 'echo "$ITEM"' >> /dev/null
|
||||||
|
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||||
|
|
||||||
|
for x in $NORMALTESTFILES
|
||||||
|
do
|
||||||
|
assertTrue "[ -e $JOBLOG/$x ]"
|
||||||
|
done
|
||||||
|
|
||||||
|
rename-ppss-dir $FUNCNAME
|
||||||
|
}
|
||||||
|
|
||||||
|
getStatusOfJob () {
|
||||||
|
|
||||||
|
EXPECTED="$1"
|
||||||
|
|
||||||
|
if [ "$EXPECTED" == "SUCCESS" ]
|
||||||
|
then
|
||||||
|
./ppss.sh -f "$INPUTFILENORMAL" -c 'echo ' >> /dev/null
|
||||||
|
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||||
|
elif [ "$EXPECTED" == "FAILURE" ]
|
||||||
|
then
|
||||||
|
./ppss.sh -f "$INPUTFILENORMAL" -c 'thiscommandfails ' >> /dev/null
|
||||||
|
assertEquals "PPSS did not execute properly." 0 "$?"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for x in $NORMALTESTFILES
|
||||||
|
do
|
||||||
|
RES=`grep "Status:" "$JOBLOG/$x"`
|
||||||
|
STATUS=`echo "$RES" | awk '{ print $2 }'`
|
||||||
|
assertEquals "FAILED WITH STATUS $STATUS." "$EXPECTED" "$STATUS"
|
||||||
|
done
|
||||||
|
|
||||||
|
rename-ppss-dir "$FUNCNAME-$EXPECTED"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
testErrorHandlingOK () {
|
||||||
|
|
||||||
|
getStatusOfJob SUCCESS
|
||||||
|
}
|
||||||
|
|
||||||
|
testErrorHandlingFAIL () {
|
||||||
|
|
||||||
|
getStatusOfJob FAILURE
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
. ./shunit2
|
50
ppss.sh
50
ppss.sh
|
@ -38,7 +38,7 @@ trap 'kill_process; ' INT
|
||||||
|
|
||||||
# Setting some vars.
|
# Setting some vars.
|
||||||
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
||||||
SCRIPT_VERSION="2.41"
|
SCRIPT_VERSION="2.45"
|
||||||
|
|
||||||
# The first argument to this script can be a mode.
|
# The first argument to this script can be a mode.
|
||||||
MODES="node start config stop pause continue deploy status erase kill"
|
MODES="node start config stop pause continue deploy status erase kill"
|
||||||
|
@ -438,7 +438,7 @@ do
|
||||||
shift 2
|
shift 2
|
||||||
;;
|
;;
|
||||||
--command|-c )
|
--command|-c )
|
||||||
COMMAND=$2
|
COMMAND="$2"
|
||||||
if [ "$MODE" == "config" ]
|
if [ "$MODE" == "config" ]
|
||||||
then
|
then
|
||||||
COMMAND=\'$COMMAND\'
|
COMMAND=\'$COMMAND\'
|
||||||
|
@ -1085,7 +1085,7 @@ escape_item () {
|
||||||
sed s/\\|/\\\\\\\\\\\\\\|/g | \
|
sed s/\\|/\\\\\\\\\\\\\\|/g | \
|
||||||
sed s/\&/\\\\\\\\\\\\\\&/g | \
|
sed s/\&/\\\\\\\\\\\\\\&/g | \
|
||||||
sed s/\;/\\\\\\\\\\\\\\;/g | \
|
sed s/\;/\\\\\\\\\\\\\\;/g | \
|
||||||
sed s/\\\//\\\\\\\\\\\\\\_/g | \
|
sed s/\\\//\\\\\\\\\\\\\\ /g | \
|
||||||
sed s/\(/\\\\\\\\\\(/g | \
|
sed s/\(/\\\\\\\\\\(/g | \
|
||||||
sed s/\)/\\\\\\\\\\)/g `
|
sed s/\)/\\\\\\\\\\)/g `
|
||||||
}
|
}
|
||||||
|
@ -1093,7 +1093,13 @@ escape_item () {
|
||||||
download_item () {
|
download_item () {
|
||||||
|
|
||||||
ITEM="$1"
|
ITEM="$1"
|
||||||
ITEM_NO_PATH=`basename "$ITEM"`
|
if [ -e "$ITEM" ]
|
||||||
|
then
|
||||||
|
ITEM_NO_PATH=`basename "$ITEM"`
|
||||||
|
else
|
||||||
|
escape_item "$ITEM"
|
||||||
|
ITEM_NO_PATH="$ITEM_ESCAPED"
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$TRANSFER_TO_SLAVE" == "1" ]
|
if [ "$TRANSFER_TO_SLAVE" == "1" ]
|
||||||
then
|
then
|
||||||
|
@ -1170,6 +1176,7 @@ lock_item () {
|
||||||
sed s/\\\//\\\\\\ /g | \
|
sed s/\\\//\\\\\\ /g | \
|
||||||
sed s/\\ /\\\\\\\\\\\\\\ /g | \
|
sed s/\\ /\\\\\\\\\\\\\\ /g | \
|
||||||
sed s/\\'/\\\\\\\\\\\\\\'/g | \
|
sed s/\\'/\\\\\\\\\\\\\\'/g | \
|
||||||
|
sed s/\\\//\\\\\\\\\\\\\\ /g | \
|
||||||
sed s/\&/\\\\\\\\\\\\\\&/g | \
|
sed s/\&/\\\\\\\\\\\\\\&/g | \
|
||||||
sed s/\;/\\\\\\\\\\\\\\;/g | \
|
sed s/\;/\\\\\\\\\\\\\\;/g | \
|
||||||
sed s/\(/\\\\\\\\\\(/g | \
|
sed s/\(/\\\\\\\\\\(/g | \
|
||||||
|
@ -1236,8 +1243,9 @@ get_all_items () {
|
||||||
((count++))
|
((count++))
|
||||||
done
|
done
|
||||||
|
|
||||||
|
exec 10>&-
|
||||||
|
|
||||||
fi
|
fi
|
||||||
exec 10>&-
|
|
||||||
|
|
||||||
SIZE_OF_ARRAY="${#ARRAY[@]}"
|
SIZE_OF_ARRAY="${#ARRAY[@]}"
|
||||||
if [ "$SIZE_OF_ARRAY" -le "0" ]
|
if [ "$SIZE_OF_ARRAY" -le "0" ]
|
||||||
|
@ -1347,12 +1355,21 @@ elapsed () {
|
||||||
commando () {
|
commando () {
|
||||||
|
|
||||||
ITEM="$1"
|
ITEM="$1"
|
||||||
DIRNAME=`dirname "$ITEM"`
|
|
||||||
ITEM_NO_PATH=`basename "$ITEM"`
|
|
||||||
OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_NO_PATH"
|
|
||||||
|
|
||||||
# This VAR can be used in scripts or command lines.
|
if [ -e "$ITEM" ]
|
||||||
OUTPUT_FILE="$ITEM_NO_PATH"
|
then
|
||||||
|
DIRNAME=`dirname "$ITEM"`
|
||||||
|
ITEM_NO_PATH=`basename "$ITEM"`
|
||||||
|
escape_item "$ITEM_NO_PATH"
|
||||||
|
OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_ESCAPED"
|
||||||
|
# This VAR can be used in scripts or command lines.
|
||||||
|
OUTPUT_FILE="$ITEM_ESCAPED"
|
||||||
|
else
|
||||||
|
DIRNAME=""
|
||||||
|
escape_item "$ITEM"
|
||||||
|
ITEM_NO_PATH="$ITEM_ESCAPED"
|
||||||
|
OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_NO_PATH"
|
||||||
|
fi
|
||||||
|
|
||||||
log DEBUG "Processing item $ITEM"
|
log DEBUG "Processing item $ITEM"
|
||||||
|
|
||||||
|
@ -1373,6 +1390,8 @@ commando () {
|
||||||
LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g`
|
LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g`
|
||||||
ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
|
ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
|
||||||
|
|
||||||
|
OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$LOG_FILE_NAME"
|
||||||
|
|
||||||
mkdir -p "$OUTPUT_DIR"
|
mkdir -p "$OUTPUT_DIR"
|
||||||
|
|
||||||
does_file_exist "$ITEM_LOG_FILE"
|
does_file_exist "$ITEM_LOG_FILE"
|
||||||
|
@ -1401,7 +1420,7 @@ commando () {
|
||||||
ERROR="$?"
|
ERROR="$?"
|
||||||
AFTER="$(date +%s)"
|
AFTER="$(date +%s)"
|
||||||
else
|
else
|
||||||
EXECME='$COMMAND"$ITEM" >> "$ITEM_LOG_FILE" 2>&1'
|
EXECME='$COMMAND$ITEM >> "$ITEM_LOG_FILE" 2>&1'
|
||||||
BEFORE="$(date +%s)"
|
BEFORE="$(date +%s)"
|
||||||
eval "$EXECME"
|
eval "$EXECME"
|
||||||
ERROR="$?"
|
ERROR="$?"
|
||||||
|
@ -1430,11 +1449,10 @@ commando () {
|
||||||
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
NEWDIR="$REMOTE_OUTPUT_DIR/$DIRNAME"
|
escape_item "$DIRNAME"
|
||||||
escape_item "$NEWDIR"
|
ITEM_OUTPUT_DIR="$REMOTE_OUTPUT_DIR/$ITEM_ESCAPED"
|
||||||
DIR_ESCAPED="$ITEM_ESCAPED"
|
|
||||||
|
|
||||||
exec_cmd "mkdir -p $DIR_ESCAPED"
|
exec_cmd "mkdir -p $ITEM_OUTPUT_DIR"
|
||||||
if [ "$DIRNAME" == "." ]
|
if [ "$DIRNAME" == "." ]
|
||||||
then
|
then
|
||||||
DIRNAME=""
|
DIRNAME=""
|
||||||
|
@ -1510,9 +1528,11 @@ listen_for_job () {
|
||||||
disown
|
disown
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
get_global_lock
|
||||||
SIZE_OF_ARRAY="${#ARRAY[@]}"
|
SIZE_OF_ARRAY="${#ARRAY[@]}"
|
||||||
ARRAY_POINTER=`cat $ARRAY_POINTER_FILE`
|
ARRAY_POINTER=`cat $ARRAY_POINTER_FILE`
|
||||||
PERCENT=$((100 * $ARRAY_POINTER / $SIZE_OF_ARRAY ))
|
PERCENT=$((100 * $ARRAY_POINTER / $SIZE_OF_ARRAY ))
|
||||||
|
release_global_lock
|
||||||
if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
|
if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
|
||||||
then
|
then
|
||||||
log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."
|
log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."
|
||||||
|
|
Loading…
Reference in New Issue