diff --git a/branches/distributed-ppss/ppss-test.sh b/branches/distributed-ppss/ppss-test.sh new file mode 100755 index 0000000..de7b2f3 --- /dev/null +++ b/branches/distributed-ppss/ppss-test.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +DEBUG="$1" +VERSION=2.45 + +cleanup () { + + for x in $REMOVEFILES + do + if [ -e ./$x ] + then + rm -r ./$x + fi + done +} + +oneTimeSetUp () { + + NORMALTESTFILES=`echo test-{a..z}` + SPECIALTESTFILES="\'file-!@#$%^&*()_+=-0987654321~\' \'file-/\<>?:;'{}[]\' file-/\/\:\/!@#$%^&*()_+=-0987654321~ file-/\<>?:;'{}[] http://www.google.nl ftp://storage.nl" + JOBLOG=./ppss/job_log + INPUTFILENORMAL=test-normal.input + INPUTFILESPECIAL=test-special.input + LOCALOUTPUT=ppss/PPSS_LOCAL_OUTPUT + + REMOVEFILES="$INPUTFILENORMAL $INPUTFILESPECIAL ppss test-ppss-*" + + cleanup + + for x in $NORMALTESTFILES + do + echo "$x" >> "$INPUTFILENORMAL" + done + + for x in $SPECIALTESTFILES + do + echo $x >> "$INPUTFILESPECIAL" + done +} + +testVersion () { + + RES=`./ppss.sh -v` + + for x in $RES + do + echo "$x" | grep [0-9] >> /dev/null + if [ "$?" == "0" ] + then + assertEquals "Version mismatch!" "$VERSION" "$x" + fi + done +} + +rename-ppss-dir () { + + TEST="$1" + + if [ -e "ppss" ] && [ -d "ppss" ] && [ ! -z "$TEST" ] + then + mv ppss test-ppss-"$TEST" + fi +} + +oneTimeTearDown () { + + if [ ! "$DEBUG" == "debug" ] + then + cleanup + fi +} + +testSpecialCharacterHandling () { + + RES=$( { ./ppss.sh -f "$INPUTFILESPECIAL" -c 'echo ' >> /dev/null ; } 2>&1 ) + assertEquals "PPSS did not execute properly." 0 "$?" + + assertNull "PPSS retured some errors..." "$RES" + if [ ! "$?" == "0" ] + then + echo "RES IS $RES" + fi + + RES=`find ppss/PPSS_LOCAL_OUTPUT | wc -l` + assertEquals "To many lock files..." "7" "$RES" + + RES1=`ls -1 $JOBLOG` + RES2=`ls -1 $LOCALOUTPUT` + + assertEquals "RES1 $RES1 is not the same as RES2 $RES2" "$RES1" "$RES2" + + rename-ppss-dir $FUNCNAME +} + +testExistLogFiles () { + + ./ppss.sh -f "$INPUTFILENORMAL" -c 'echo "$ITEM"' >> /dev/null + assertEquals "PPSS did not execute properly." 0 "$?" + + for x in $NORMALTESTFILES + do + assertTrue "[ -e $JOBLOG/$x ]" + done + + rename-ppss-dir $FUNCNAME +} + +getStatusOfJob () { + + EXPECTED="$1" + + if [ "$EXPECTED" == "SUCCESS" ] + then + ./ppss.sh -f "$INPUTFILENORMAL" -c 'echo ' >> /dev/null + assertEquals "PPSS did not execute properly." 0 "$?" + elif [ "$EXPECTED" == "FAILURE" ] + then + ./ppss.sh -f "$INPUTFILENORMAL" -c 'thiscommandfails ' >> /dev/null + assertEquals "PPSS did not execute properly." 0 "$?" + fi + + for x in $NORMALTESTFILES + do + RES=`grep "Status:" "$JOBLOG/$x"` + STATUS=`echo "$RES" | awk '{ print $2 }'` + assertEquals "FAILED WITH STATUS $STATUS." "$EXPECTED" "$STATUS" + done + + rename-ppss-dir "$FUNCNAME-$EXPECTED" +} + + +testErrorHandlingOK () { + + getStatusOfJob SUCCESS +} + +testErrorHandlingFAIL () { + + getStatusOfJob FAILURE +} + + + + +. ./shunit2 diff --git a/branches/distributed-ppss/ppss.sh b/branches/distributed-ppss/ppss.sh index 9d1c749..b4528c8 100755 --- a/branches/distributed-ppss/ppss.sh +++ b/branches/distributed-ppss/ppss.sh @@ -38,7 +38,7 @@ trap 'kill_process; ' INT # Setting some vars. SCRIPT_NAME="Distributed Parallel Processing Shell Script" -SCRIPT_VERSION="2.41" +SCRIPT_VERSION="2.45" # The first argument to this script can be a mode. MODES="node start config stop pause continue deploy status erase kill" @@ -438,7 +438,7 @@ do shift 2 ;; --command|-c ) - COMMAND=$2 + COMMAND="$2" if [ "$MODE" == "config" ] then COMMAND=\'$COMMAND\' @@ -1085,7 +1085,7 @@ escape_item () { sed s/\\|/\\\\\\\\\\\\\\|/g | \ sed s/\&/\\\\\\\\\\\\\\&/g | \ sed s/\;/\\\\\\\\\\\\\\;/g | \ - sed s/\\\//\\\\\\\\\\\\\\_/g | \ + sed s/\\\//\\\\\\\\\\\\\\ /g | \ sed s/\(/\\\\\\\\\\(/g | \ sed s/\)/\\\\\\\\\\)/g ` } @@ -1093,7 +1093,13 @@ escape_item () { download_item () { ITEM="$1" - ITEM_NO_PATH=`basename "$ITEM"` + if [ -e "$ITEM" ] + then + ITEM_NO_PATH=`basename "$ITEM"` + else + escape_item "$ITEM" + ITEM_NO_PATH="$ITEM_ESCAPED" + fi if [ "$TRANSFER_TO_SLAVE" == "1" ] then @@ -1170,6 +1176,7 @@ lock_item () { sed s/\\\//\\\\\\ /g | \ sed s/\\ /\\\\\\\\\\\\\\ /g | \ sed s/\\'/\\\\\\\\\\\\\\'/g | \ + sed s/\\\//\\\\\\\\\\\\\\ /g | \ sed s/\&/\\\\\\\\\\\\\\&/g | \ sed s/\;/\\\\\\\\\\\\\\;/g | \ sed s/\(/\\\\\\\\\\(/g | \ @@ -1235,9 +1242,10 @@ get_all_items () { ARRAY[$count]=$LINE ((count++)) done + + exec 10>&- fi - exec 10>&- SIZE_OF_ARRAY="${#ARRAY[@]}" if [ "$SIZE_OF_ARRAY" -le "0" ] @@ -1347,12 +1355,21 @@ elapsed () { commando () { ITEM="$1" - DIRNAME=`dirname "$ITEM"` - ITEM_NO_PATH=`basename "$ITEM"` - OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_NO_PATH" - # This VAR can be used in scripts or command lines. - OUTPUT_FILE="$ITEM_NO_PATH" + if [ -e "$ITEM" ] + then + DIRNAME=`dirname "$ITEM"` + ITEM_NO_PATH=`basename "$ITEM"` + escape_item "$ITEM_NO_PATH" + OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_ESCAPED" + # This VAR can be used in scripts or command lines. + OUTPUT_FILE="$ITEM_ESCAPED" + else + DIRNAME="" + escape_item "$ITEM" + ITEM_NO_PATH="$ITEM_ESCAPED" + OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_NO_PATH" + fi log DEBUG "Processing item $ITEM" @@ -1373,6 +1390,8 @@ commando () { LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g` ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME" + OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$LOG_FILE_NAME" + mkdir -p "$OUTPUT_DIR" does_file_exist "$ITEM_LOG_FILE" @@ -1401,7 +1420,7 @@ commando () { ERROR="$?" AFTER="$(date +%s)" else - EXECME='$COMMAND"$ITEM" >> "$ITEM_LOG_FILE" 2>&1' + EXECME='$COMMAND$ITEM >> "$ITEM_LOG_FILE" 2>&1' BEFORE="$(date +%s)" eval "$EXECME" ERROR="$?" @@ -1430,11 +1449,10 @@ commando () { fi - NEWDIR="$REMOTE_OUTPUT_DIR/$DIRNAME" - escape_item "$NEWDIR" - DIR_ESCAPED="$ITEM_ESCAPED" + escape_item "$DIRNAME" + ITEM_OUTPUT_DIR="$REMOTE_OUTPUT_DIR/$ITEM_ESCAPED" - exec_cmd "mkdir -p $DIR_ESCAPED" + exec_cmd "mkdir -p $ITEM_OUTPUT_DIR" if [ "$DIRNAME" == "." ] then DIRNAME="" @@ -1510,9 +1528,11 @@ listen_for_job () { disown fi + get_global_lock SIZE_OF_ARRAY="${#ARRAY[@]}" ARRAY_POINTER=`cat $ARRAY_POINTER_FILE` PERCENT=$((100 * $ARRAY_POINTER / $SIZE_OF_ARRAY )) + release_global_lock if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ] then log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."