Significant bug fixing and testing. Updated to 2.45

branches/distributed-ppss/ppss-test.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+
+DEBUG="$1"
+VERSION=2.45
+
+cleanup () {
+
+    for x in $REMOVEFILES
+    do
+        if [ -e ./$x ]
+        then
+            rm -r ./$x
+        fi
+    done
+}
+
+oneTimeSetUp () {
+
+	NORMALTESTFILES=`echo test-{a..z}`
+    SPECIALTESTFILES="\'file-!@#$%^&*()_+=-0987654321~\' \'file-/\<>?:;'{}[]\' file-/\/\:\/!@#$%^&*()_+=-0987654321~ file-/\<>?:;'{}[] http://www.google.nl ftp://storage.nl"
+	JOBLOG=./ppss/job_log
+	INPUTFILENORMAL=test-normal.input
+    INPUTFILESPECIAL=test-special.input
+    LOCALOUTPUT=ppss/PPSS_LOCAL_OUTPUT
+
+	REMOVEFILES="$INPUTFILENORMAL $INPUTFILESPECIAL ppss test-ppss-*"
+
+    cleanup
+
+	for x in $NORMALTESTFILES
+	do
+		echo "$x" >> "$INPUTFILENORMAL"
+	done
+
+    for x in $SPECIALTESTFILES
+    do
+        echo $x >> "$INPUTFILESPECIAL"
+    done
+}
+
+testVersion () {
+
+    RES=`./ppss.sh -v`
+    
+    for x in $RES
+    do
+        echo "$x" | grep [0-9] >> /dev/null
+        if [ "$?" == "0" ]
+        then
+            assertEquals "Version mismatch!" "$VERSION" "$x"
+        fi
+    done
+}
+
+rename-ppss-dir () {
+
+	TEST="$1"
+
+	if [ -e "ppss" ] && [ -d "ppss" ] && [ ! -z "$TEST" ]
+	then
+		mv ppss test-ppss-"$TEST"
+	fi
+}
+
+oneTimeTearDown () {
+
+	if [ ! "$DEBUG" == "debug" ]
+	then
+        cleanup 
+    fi
+}
+
+testSpecialCharacterHandling () {
+
+    RES=$( { ./ppss.sh -f "$INPUTFILESPECIAL" -c 'echo ' >> /dev/null ; } 2>&1 )  
+	assertEquals "PPSS did not execute properly." 0 "$?"
+
+    assertNull "PPSS retured some errors..." "$RES"
+    if [ ! "$?" == "0" ]
+    then
+        echo "RES IS $RES"
+    fi
+
+    RES=`find ppss/PPSS_LOCAL_OUTPUT | wc -l`
+    assertEquals "To many lock files..." "7" "$RES"
+
+    RES1=`ls -1 $JOBLOG`
+    RES2=`ls -1 $LOCALOUTPUT`
+
+    assertEquals "RES1 $RES1 is not the same as RES2 $RES2" "$RES1" "$RES2"
+
+    rename-ppss-dir $FUNCNAME
+}
+
+testExistLogFiles () {
+
+	./ppss.sh -f "$INPUTFILENORMAL" -c 'echo "$ITEM"' >> /dev/null
+	assertEquals "PPSS did not execute properly." 0 "$?"
+
+	for x in $NORMALTESTFILES
+	do
+		assertTrue "[ -e $JOBLOG/$x ]"
+	done
+
+	rename-ppss-dir $FUNCNAME
+}
+
+getStatusOfJob () {
+
+	EXPECTED="$1"
+
+	if [ "$EXPECTED" == "SUCCESS" ]
+	then
+		./ppss.sh -f "$INPUTFILENORMAL" -c 'echo ' >> /dev/null
+        	assertEquals "PPSS did not execute properly." 0 "$?"
+	elif [ "$EXPECTED" == "FAILURE" ]
+	then
+		./ppss.sh -f "$INPUTFILENORMAL" -c 'thiscommandfails ' >> /dev/null
+        	assertEquals "PPSS did not execute properly." 0 "$?"
+	fi
+
+	for x in $NORMALTESTFILES
+	do
+		RES=`grep "Status:" "$JOBLOG/$x"`
+        STATUS=`echo "$RES" | awk '{ print $2 }'`
+        assertEquals "FAILED WITH STATUS $STATUS." "$EXPECTED" "$STATUS"
+    done
+
+	rename-ppss-dir "$FUNCNAME-$EXPECTED"
+}
+
+
+testErrorHandlingOK () {
+
+	getStatusOfJob SUCCESS
+}
+
+testErrorHandlingFAIL () {
+
+	getStatusOfJob FAILURE
+}
+
+
+
+
+. ./shunit2

branches/distributed-ppss/ppss.sh

@@ -38,7 +38,7 @@ trap 'kill_process; ' INT
 
 # Setting some vars. 
 SCRIPT_NAME="Distributed Parallel Processing Shell Script"
-SCRIPT_VERSION="2.41"
+SCRIPT_VERSION="2.45"
 
 # The first argument to this script can be a mode.
 MODES="node start config stop pause continue deploy status erase kill"
@@ -438,7 +438,7 @@ do
                         shift 2
                         ;;
           --command|-c ) 
-                        COMMAND=$2
+                        COMMAND="$2"
                         if [ "$MODE" == "config" ]
                         then
                             COMMAND=\'$COMMAND\'
@@ -1085,7 +1085,7 @@ escape_item () {
             sed s/\\|/\\\\\\\\\\\\\\|/g | \
             sed s/\&/\\\\\\\\\\\\\\&/g | \
             sed s/\;/\\\\\\\\\\\\\\;/g | \
-            sed s/\\\//\\\\\\\\\\\\\\_/g | \
+            sed s/\\\//\\\\\\\\\\\\\\ /g | \
             sed s/\(/\\\\\\\\\\(/g | \
             sed s/\)/\\\\\\\\\\)/g ` 
 }
@@ -1093,7 +1093,13 @@ escape_item () {
 download_item () {
 
     ITEM="$1"
+    if [ -e "$ITEM" ]
+    then
         ITEM_NO_PATH=`basename "$ITEM"`
+    else
+        escape_item "$ITEM"
+        ITEM_NO_PATH="$ITEM_ESCAPED"
+    fi
 
     if [ "$TRANSFER_TO_SLAVE" == "1" ]
     then
@@ -1170,6 +1176,7 @@ lock_item () {
         sed s/\\\//\\\\\\ /g | \
         sed s/\\ /\\\\\\\\\\\\\\ /g | \
         sed s/\\'/\\\\\\\\\\\\\\'/g | \
+        sed s/\\\//\\\\\\\\\\\\\\ /g | \
         sed s/\&/\\\\\\\\\\\\\\&/g | \
         sed s/\;/\\\\\\\\\\\\\\;/g | \
         sed s/\(/\\\\\\\\\\(/g | \
@@ -1236,9 +1243,10 @@ get_all_items () {
             ((count++))
         done
 
-    fi
         exec 10>&-
   
+    fi
+
     SIZE_OF_ARRAY="${#ARRAY[@]}"
     if [ "$SIZE_OF_ARRAY" -le "0" ]
     then
@@ -1347,12 +1355,21 @@ elapsed () {
 commando () {
 
     ITEM="$1"
+
+    if [ -e "$ITEM" ]
+    then
         DIRNAME=`dirname "$ITEM"`
         ITEM_NO_PATH=`basename "$ITEM"`
-    OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_NO_PATH"
+        escape_item "$ITEM_NO_PATH"
-
+        OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_ESCAPED"
         # This VAR can be used in scripts or command lines.
-    OUTPUT_FILE="$ITEM_NO_PATH"
+        OUTPUT_FILE="$ITEM_ESCAPED"
+    else
+        DIRNAME=""
+        escape_item "$ITEM"
+        ITEM_NO_PATH="$ITEM_ESCAPED"
+        OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$ITEM_NO_PATH"
+    fi
 
     log DEBUG "Processing item $ITEM"
 
@@ -1373,6 +1390,8 @@ commando () {
     LOG_FILE_NAME=`echo "$ITEM" | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g`
     ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
 
+    OUTPUT_DIR=$PPSS_LOCAL_OUTPUT/"$LOG_FILE_NAME"
+
     mkdir -p "$OUTPUT_DIR"
 
     does_file_exist "$ITEM_LOG_FILE"
@@ -1401,7 +1420,7 @@ commando () {
             ERROR="$?"
             AFTER="$(date +%s)"
         else
-            EXECME='$COMMAND"$ITEM" >> "$ITEM_LOG_FILE" 2>&1'
+            EXECME='$COMMAND$ITEM  >> "$ITEM_LOG_FILE" 2>&1'
             BEFORE="$(date +%s)"
             eval "$EXECME"
             ERROR="$?"
@@ -1430,11 +1449,10 @@ commando () {
 
         fi
 
-        NEWDIR="$REMOTE_OUTPUT_DIR/$DIRNAME"
+        escape_item "$DIRNAME"
-        escape_item "$NEWDIR"
+        ITEM_OUTPUT_DIR="$REMOTE_OUTPUT_DIR/$ITEM_ESCAPED"
-        DIR_ESCAPED="$ITEM_ESCAPED"
 
-        exec_cmd "mkdir -p $DIR_ESCAPED"
+        exec_cmd "mkdir -p $ITEM_OUTPUT_DIR"
         if [ "$DIRNAME" == "." ]
         then
             DIRNAME=""
@@ -1510,9 +1528,11 @@ listen_for_job () {
             disown
         fi
 
+        get_global_lock
         SIZE_OF_ARRAY="${#ARRAY[@]}"
         ARRAY_POINTER=`cat $ARRAY_POINTER_FILE`
         PERCENT=$((100 * $ARRAY_POINTER / $SIZE_OF_ARRAY ))
+        release_global_lock
         if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
         then
             log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."